From b8868e363fdd9db1b5a706f584c64da532210465 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 31 Jul 2026 21:49:53 +0000 Subject: [PATCH 01/26] docs: add AWS Aurora blue green design specs --- ...ra-bgd-cluster-simulator-testing-design.md | 356 +++++++++++++ ...nfiguration-runtime-cluster-sync-design.md | 417 ++++++++++++++++ ...026-07-31-aurora-bgd-monitor-fsm-design.md | 468 ++++++++++++++++++ 3 files changed, 1241 insertions(+) create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md new file mode 100644 index 0000000000..f3fda79484 --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -0,0 +1,356 @@ +# Aurora BGD Cluster Simulator and Testing Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) + +## Spec Boundary + +This specification owns the simulator capabilities and executable test coverage +needed to validate the other two specifications. It does not redefine public +configuration or FSM behavior. When a test expectation depends on those +contracts, the corresponding sibling specification is authoritative. + +## 1. Purpose + +Provide deterministic coverage for Aurora MySQL blue/green deployments without +requiring live AWS infrastructure. The test environment must be able to drive +the complete BGD observation sequence, publish target Aurora membership, +simulate member renaming, inject failures, observe monitor probes, and verify +ProxySQL runtime and routing effects. + +The design must preserve both existing regression suites: + +- the ordinary Aurora cluster simulator continues validating Aurora role, + membership, lag, failover, and autopurge behavior; +- the RDS Multi-AZ BGD suite continues validating its existing instance-based + mapping and reader-cleanup FSM. + +## 2. Architecture Decision + +Aurora BGD FSM tests extend the existing interactive RDS BGD simulator used by +`cluster_sim_rds_bgd-g1`. They do not add BGD sequencing to the legacy JSON +Aurora simulator. + +This is the preferred design because the RDS BGD simulator already provides: + +- per-backend `mysql.rds_topology` responses; +- empty, absent, and error topology modes; +- read-only controls; +- ordered probe logging; +- fixed AWS-style hostname-to-loopback mappings; +- TAP-driven, phase-by-phase transitions; +- the build and CI integration required for BGD tests. + +Aurora adds a target-cluster membership service and Aurora-specific TAP helpers +on top of that base. Shared topology, transaction, endpoint, TLS, read-only, +probe-wait, and cleanup behavior remains common. + +### Alternatives not selected + +1. Extending the JSON Aurora simulator with a long sequence of BGD states would + overload a batch-oriented two-state model and make late entry, retry, + rollback, and concurrent timing difficult to control. +2. Creating a third standalone simulator group would duplicate the existing RDS + BGD topology service, host maps, helpers, build flavor, and CI wiring. +3. Using only unit tests would not exercise monitor threads, DNS pins, + connection-pool drains, runtime status publication, or config reloads. + +## 3. Simulator Service Contract + +The `TEST_RDS_BGD` SQLite3-server flavor remains the executable backend for the +interactive suite. It is extended to recognize the Aurora membership query in +addition to the existing topology and read-only queries. + +All simulated responses are keyed by the backend address and port on which the +SQLite3 server accepted the monitor connection. This preserves isolation when +multiple deployments or multiple target endpoints are active concurrently. + +### 3.1 Existing topology service + +The existing service remains authoritative for: + +```text +mysql.rds_topology table presence +SOURCE and TARGET rows +deployment fingerprint fields +AWS status strings +configured topology errors +ordered table-check and metadata probe logs +``` + +Aurora TARGET rows contain a cluster endpoint. Multi-AZ TARGET rows continue to +contain an instance endpoint. The simulator must not infer deployment type from +test configuration; production detection consumes the endpoint shape and the +membership-query result. + +### 3.2 Aurora membership service + +The simulator adds per-backend control and row storage for +`INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. + +Each membership row contains at least: + +```text +row_order +SERVER_ID +SESSION_ID +LAST_UPDATE_TIMESTAMP +IS_CURRENT +CPU +REPLICA_LAG_IN_MILLISECONDS +``` + +`SESSION_ID='MASTER_SESSION_ID'` identifies the writer. Other current rows are +readers. `row_order` makes response ordering deterministic while allowing tests +to prove that ProxySQL does not rely on writer-first or lexical membership +ordering. + +The control state supports: + +- a successful complete membership result; +- a successful writer-only result; +- an intentionally incomplete result; +- an empty result; +- table absence/error 1146; +- an arbitrary MySQL error code and message. + +Membership updates are atomic per supplied set of backends. A test must not +expose a partially rewritten snapshot unless it explicitly selects the +incomplete-result mode. + +### 3.3 Probe log + +The existing BGD probe log adds an Aurora-membership probe kind. Every topology +table check, topology metadata query, and target membership query records: + +```text +monotonic sequence +accepted backend IP +accepted backend port +probe kind +TLS state +``` + +Tests use the log to verify probe destination, ordering, cadence class, TLS, +probe-pin retention, and return to canonical probing. They must not use fixed +sleeps when an observable probe or runtime state can serve as the wait +condition. + +### 3.4 Error isolation + +Topology and membership errors are independent. A scenario can publish valid +topology with failed membership, or valid membership with failed topology. +Clearing one error source must not silently clear the other. + +Simulator cleanup removes topology, membership, read-only, and probe-log state +in one operation so each TAP binary begins from a known baseline. + +## 4. TAP Helper Model + +Aurora-specific helpers extend, rather than fork, the RDS BGD helper model. +They provide test-facing representations for: + +- a blue Aurora cluster and its canonical instance endpoints; +- a target cluster endpoint; +- one target writer and zero or more target readers; +- stable member session identities; +- pre-rename green `SERVER_ID` values; +- post-rename canonical `SERVER_ID` values; +- explicit and automatic Aurora hostgroup configuration; +- expected runtime `bgd_status` values. + +The helper API exposes operations equivalent to: + +```text +publish topology status +publish target membership snapshot +publish renamed membership snapshot +publish empty/absent topology +inject topology or membership error +record and wait for probes +query Aurora runtime row and bgd_status +query runtime server placement and status +query connection-pool state +open backend traffic and identify accepted simulator IP +clean up ProxySQL and simulator state +``` + +Helper methods perform control operations only. Individual test scenarios own +the sequence of AWS observations and all expected ProxySQL outcomes. + +## 5. Endpoint and DNS Model + +The `cluster_sim_rds_bgd-g1` fixed host map is extended with Aurora cluster +endpoints and member endpoints. Each hostname maps to a distinct loopback +address accepted by the same SQLite3 server. + +The map includes, per test cluster: + +- canonical blue writer and reader instance names; +- the green target cluster endpoint returned by `mysql.rds_topology`; +- green-suffixed target writer and reader instance names; +- canonical post-rename member names; +- a second target deployment for repeated-switchover coverage; +- additional clusters for concurrency coverage. + +Green and canonical names used for identity-renaming tests may resolve to the +same target-member loopback IP where that models AWS's post-processing rename. +Canonical blue DNS remains static in the container; DNS-cache pin behavior is +verified through accepted backend addresses and explicit pin removal, not by +claiming that the simulator reproduces mutable Route 53 propagation. + +## 6. Test Layers + +### 6.1 Schema and unit coverage + +Fast tests cover deterministic contracts that do not require monitor threads: + +- configuration and runtime table definitions and column order; +- paired-NULL and hostgroup-conflict validation; +- SQL NULL preservation in bind/extract helpers; +- config-file import and export; +- disk schema upgrade defaults; +- runtime `bgd_status` string mapping; +- configured-column cluster query and checksum projection; +- exclusion of `bgd_status` from SAVE and cluster synchronization; +- Aurora simulator JSON parsing of the two optional green hostgroups. + +### 6.2 Admin and cluster-sync integration coverage + +Admin/TAP tests cover: + +- memory-to-runtime and runtime-to-memory round trips; +- runtime-to-disk and config-file round trips; +- new runtime rows starting at `NONE`; +- reload preservation of `bgd_status` and active worker state; +- removal and deactivation behavior; +- peer synchronization of both green hostgroups, including NULL; +- proof that peers retain independent node-local `bgd_status` values; +- unchanged RDS Multi-AZ BGD cluster synchronization. + +### 6.3 Interactive Aurora BGD coverage + +The interactive suite covers the observable behavior of the monitor/FSM spec: + +1. `AVAILABLE` discovers exactly one writer and every reader, resolves each + member, and establishes only the probe state required by the design. +2. Automatic mode with NULL green hostgroups maps all target members without + generating green hostgroups or `mysql_servers` rows. +3. Explicit mode uses configured green hostgroups as staging pools but still + treats `REPLICA_HOST_STATUS` as membership truth. +4. Initiated and in-progress observations retain the complete snapshot and + engage BGD/read-only-monitor protection. +5. The first post-processing observation pins and drains each writer/reader pair + once; repeated observations retry only incomplete member actions. +6. Member `SERVER_ID` rename preserves reader identity through stable + `SESSION_ID` values and does not change cached target IPs. +7. The first TARGET `SWITCHOVER_COMPLETED` removes all traffic and probe pins, + performs cleanup once, and publishes `bgd_status='SWITCHOVER_COMPLETED'`. +8. Repeated completed rows are no-ops; successful topology drain changes the + runtime status to `NONE` and rearms discovery. +9. A different deployment fingerprint can rearm from the terminal latch without + inheriting stale members, pins, probes, or completion flags. + +### 6.4 Resilience and edge coverage + +Scenarios also cover: + +- membership row ordering and zero-reader target clusters; +- incomplete membership retaining the last complete snapshot; +- topology and membership query failures in every phase; +- rollback from initiated, in-progress, and post-processing observations; +- late entry at initiated, in-progress, post-processing, and completed states; +- config refresh and hostgroup refresh during active phases; +- worker restart/respawn with preserved fingerprint, members, IPs, pins, and + terminal latch; +- disabling automatic discovery during an active automatic deployment; +- deleting or deactivating an Aurora row during a switchover; +- explicit green-pool cleanup with ONLINE, SHUNNED, OFFLINE_SOFT, and + OFFLINE_HARD members; +- TLS selection for topology and membership probes; +- multiple concurrent Aurora deployments with independent state; +- simultaneous Aurora BGD and RDS Multi-AZ BGD deployments. + +## 7. Regression Coverage + +The existing `test_cluster_sim_aurora-t` JSON payload suite remains responsible +for ordinary Aurora behavior. Its schema accepts the two new green hostgroup +fields as optional values but old payloads remain valid and preserve their +existing results. + +The complete existing `test_rds_bgd_*` suite remains unchanged in semantics. +Aurora membership support is additive to the shared simulator and must not +alter instance TARGET handling, reader shun/unshun policy, writer fallback, or +Multi-AZ completion/drain behavior. + +Regression runs must include: + +```text +ordinary Aurora cluster simulator group +RDS Multi-AZ BGD simulator group +new Aurora BGD TAP binaries in cluster_sim_rds_bgd-g1 +configuration/unit tests +ProxySQL Cluster synchronization tests +``` + +## 8. Determinism and Timing + +Tests synchronize on observable state: runtime `bgd_status`, probe sequence, +server placement, pool counters, and successful simulator control commits. +Timeouts are derived from configured monitor intervals with bounded slack. + +Fixed sleeps are allowed only for negative assertions where no event can be +awaited directly, and must be shorter than the overall TAP timeout. Every wait +failure reports the last runtime row, relevant server/pool state, and probes +observed since the scenario checkpoint. + +Each scenario uses unique hostgroups or performs complete cleanup. Concurrent +tests use disjoint endpoints and deployment fingerprints. + +## 9. Build and CI Contract + +The existing `test_rds_bgd` build remains the focused local build. The combined +cluster-simulator build continues compiling both `TEST_AURORA` and +`TEST_RDS_BGD` support. + +New interactive TAP binaries register in `cluster_sim_rds_bgd-g1`, which keeps +the existing fixed-host injection, SQLite3-server startup, no-backend-infra +model, CI matrix discovery, and log collection. No new workflow or simulator +group is required. + +## 10. Non-Goals + +The simulator does not claim to validate: + +- AWS control-plane APIs or real AWS timing; +- mutable Route 53 propagation; +- application-level latency or packet loss; +- cross-process persistence of transient FSM state; +- behavior of Aurora versions that violate the AWS contract in the monitor/FSM + specification. + +Live-AWS evidence remains a separate validation layer for assumptions about +status ordering, member rename, writability, and DNS completion. + +## 11. Acceptance Criteria + +The simulator/testing design is satisfied when: + +1. Tests can independently control topology and target membership per backend. +2. The full Aurora BGD FSM can be driven without live AWS infrastructure. +3. Runtime status, routing, pool, DNS-pin, probe, reload, and cluster-sync + contracts have deterministic assertions. +4. Automatic and explicit modes both cover writer and all-reader membership. +5. Rename, rollback, late-entry, error, refresh, repeated, and concurrent paths + are covered. +6. Existing ordinary Aurora and RDS Multi-AZ BGD suites retain their semantics. +7. The suite runs through existing cluster-simulator build and CI plumbing with + no new infrastructure group. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md new file mode 100644 index 0000000000..d2c4ac3c7f --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -0,0 +1,417 @@ +# Aurora BGD Configuration, Runtime Status, and Cluster Sync Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft; configuration decisions locked + +**Scope:** Configuration and runtime integration of Aurora MySQL blue/green +deployment handling with the existing `mysql_aws_aurora_hostgroups` subsystem. + +**Related designs:** + +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the public configuration contract, the runtime-only +`bgd_status` contract, load/save and schema-upgrade behavior, and ProxySQL +Cluster synchronization. It treats the worker state machine as a consumer and +publisher through the interface defined here; membership discovery, routing +actions, and FSM internals belong to the monitor/FSM specification. + +## 1. Decision Summary + +Aurora blue/green handling is part of the existing Aurora monitor. It does not +require an entry in `mysql_aws_rds_bgd_hostgroups`. + +Two nullable columns are added to `mysql_aws_aurora_hostgroups` and +`runtime_mysql_aws_aurora_hostgroups`: + +```sql +green_writer_hostgroup INT DEFAULT NULL +green_reader_hostgroup INT DEFAULT NULL +``` + +The columns provide explicit staging hostgroups when configured. When both are +NULL, the existing global variable +`mysql-aws_blue_green_deployment_auto_discovery` controls automatic Aurora BGD +discovery. + +No separate Aurora BGD enable column is added. + +The runtime table adds one runtime-only observability column: + +```sql +bgd_status VARCHAR NOT NULL DEFAULT 'NONE' +``` + +It reports the Aurora worker's local BGD FSM state. It is not configuration and +is excluded from persistence and ProxySQL Cluster synchronization. + +## 2. Table Schema + +The configuration table uses the following column order: + +```text +writer_hostgroup +reader_hostgroup +green_writer_hostgroup +green_reader_hostgroup +active +aurora_port +domain_name +max_lag_ms +check_interval_ms +check_timeout_ms +writer_is_also_reader +new_reader_weight +add_lag_ms +min_lag_ms +lag_num_checks +autopurge_missing_checks +comment +``` + +The runtime table uses the same order and appends: + +```text +bgd_status +``` + +The new column definitions are: + +```sql +green_writer_hostgroup INT DEFAULT NULL + CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup >= 0), + +green_reader_hostgroup INT DEFAULT NULL + CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup >= 0) +``` + +Both columns are nullable in the configuration and runtime tables. Their NULL +state is meaningful and must be preserved by every load, save, export, import, +and cluster-synchronization path. + +The runtime table mirrors these configured fields and adds only `bgd_status`. +This design does not add an `auto_generated` column because the existing Aurora +row is the owner of both normal Aurora monitoring and BGD monitoring. It also +does not add a runtime `mode` column; effective mode remains derived from the +configured green hostgroups and the global auto-discovery variable. + +## 3. Configuration Modes + +The effective mode for an active Aurora row is determined as follows: + +| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Effective behavior | +|---|---:|---| +| Both non-NULL | `0` or `1` | Explicit Aurora BGD monitoring | +| Both NULL | `1` | Automatic Aurora BGD monitoring | +| Both NULL | `0` | Aurora monitoring without BGD discovery | +| Exactly one NULL | `0` or `1` | Invalid configuration | +| Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | + +### 3.1 Explicit mode + +A row is in explicit mode when both green hostgroups are non-NULL. + +In explicit mode: + +- BGD monitoring is enabled independently of the global auto-discovery value. +- The target cluster writer is associated with `green_writer_hostgroup`. +- All target cluster readers are associated with `green_reader_hostgroup`. +- Target membership still comes from the target cluster's + `REPLICA_HOST_STATUS`; configured green `mysql_servers` rows are not the + membership source of truth. +- The green hostgroups provide optional user-visible staging/routing pools in + addition to the internal BGD member map. + +### 3.2 Automatic mode + +A row is in automatic mode when both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime. + +In automatic mode: + +- The existing Aurora monitor detects BGD topology for the row. +- The monitor discovers the target writer and all target readers through + `REPLICA_HOST_STATUS`. +- The complete target member map and cached IPs remain internal. +- No green hostgroups or green `mysql_servers` rows are generated. +- Post-processing pinning covers the writer and every reader despite the green + hostgroup columns being NULL. + +Automatic mode therefore does not use the writer-only fallback or reader +shun/unshun policy from the Multi-AZ instance implementation. + +### 3.3 BGD disabled for the row + +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=false`, the worker continues its +existing Aurora role and lag monitoring but does not start discovery of a new +BGD deployment. + +Disabling global auto-discovery while an automatically discovered switchover is +already active must not abandon that switchover. The worker completes pin +cleanup and enters topology-drain wait before disabling discovery for the row. +The variable gates the start of new automatic BGD state machines, not safe +completion of one already in progress. + +## 4. Validation Rules + +For each `mysql_aws_aurora_hostgroups` row: + +1. `green_writer_hostgroup` and `green_reader_hostgroup` must either both be + NULL or both be non-NULL. +2. When non-NULL, all four hostgroups must be distinct: + + ```text + writer_hostgroup + reader_hostgroup + green_writer_hostgroup + green_reader_hostgroup + ``` + +3. A hostgroup assigned to one Aurora row must not conflict with any blue or + green role in another active Aurora row. +4. Existing writer-hostgroup primary-key and reader-hostgroup uniqueness rules + remain in effect. +5. Invalid rows must be rejected or excluded from runtime loading with a clear + admin error identifying the writer hostgroup and conflicting fields. + +Validation must occur before publishing the new Aurora monitor resultset so a +bad row cannot partially reconfigure a running monitor worker. + +## 5. Runtime Ownership + +`AWS_Aurora_Info`, keyed by `writer_hostgroup`, remains the runtime owner of the +cluster configuration. It is extended with optional integer fields: + +```text +green_writer_hostgroup = -1 when SQL NULL +green_reader_hostgroup = -1 when SQL NULL +``` + +The existing Aurora monitor worker remains one worker per active writer +hostgroup. The worker owns the BGD FSM and publishes each transition to the +runtime row's `bgd_status`. Its effective BGD mode is derived from: + +```text +AWS_Aurora_Info.active +green_writer_hostgroup +green_reader_hostgroup +mysql_thread___aws_blue_green_deployment_auto_discovery +``` + +No Aurora configuration is copied into `mysql_aws_rds_bgd_hostgroups`, and no +second BGD worker is started for the same Aurora writer hostgroup. + +Following RDS BGD, the Hostgroups Manager's internal Aurora table is the source +used to materialize `runtime_mysql_aws_aurora_hostgroups`. It stores the +configured fields plus `bgd_status`. Its runtime dump includes all of those +fields; paths that write back to configuration explicitly project away +`bgd_status`. + +## 6. LOAD Behavior + +`LOAD MYSQL SERVERS TO RUNTIME` must: + +1. Read both new columns from `mysql_aws_aurora_hostgroups`. +2. Preserve SQL NULL as the internal unset value rather than converting it to + hostgroup `0`. +3. Validate paired NULL/non-NULL and hostgroup-conflict rules. +4. Update or create the `AWS_Aurora_Info` entry. +5. Include both fields in the Aurora monitor resultset checksum. +6. Restart/refresh only the affected Aurora writer-hostgroup worker when either + green hostgroup changes. +7. Preserve an active BGD FSM safely across an unrelated configuration refresh. +8. Initialize `bgd_status` to `NONE` for a newly published runtime row. +9. Preserve the existing `bgd_status` when merging an existing + writer-hostgroup runtime row, following the RDS BGD table-reload behavior. + The Aurora worker separately preserves the underlying FSM state required by + the monitor/FSM specification. +10. Remove `bgd_status` with a removed runtime row; active-operation teardown + follows the existing RDS BGD worker-removal cleanup pattern. + +Changing the green hostgroups during an active switchover must not lose cached +member identities, applied DNS pins, or completion-latch state. The worker must +apply the refreshed explicit staging configuration without restarting the BGD +operation from `NONE`. + +`LOAD MYSQL VARIABLES TO RUNTIME` makes a change to +`aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. +The variable controls admission of new automatic BGD operations as described in +Section 3.3. + +## 7. Runtime Table Behavior + +`runtime_mysql_aws_aurora_hostgroups` exposes both new columns in the same +positions as the configuration table and appends the runtime-only +`bgd_status` column. + +`bgd_status` exposes the internal Aurora BGD FSM state, not merely the last raw +AWS status string. Its values are: + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +The worker updates this column as its FSM changes, following the existing RDS +BGD runtime-status mechanism wherever applicable. `SWITCHOVER_COMPLETED` is the +terminal rearm latch: all routing cleanup has already completed, and the status +remains visible until `mysql.rds_topology` drains. A successful empty/absent +topology result then changes it to `NONE`. Query or connection errors while +latched do not reset the status or repeat cleanup. + +An Aurora row that is not handling a BGD deployment reports `NONE`, including +ordinary Aurora monitoring when automatic discovery is disabled. + +Examples: + +Explicit mode: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, 11, 21, '.cluster-example.eu-north-1.rds.amazonaws.com'); +``` + +Automatic mode: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, NULL, NULL, '.cluster-example.eu-north-1.rds.amazonaws.com'); + +SET mysql-aws_blue_green_deployment_auto_discovery = 'true'; +``` + +The BGD FSM state is operational monitor state, not user configuration. +`bgd_status` provides its runtime observability without changing the +explicit/automatic mode contract defined here. + +## 8. Persistence and Synchronization + +Both green hostgroup columns must be supported by: + +- admin-memory table creation; +- runtime table creation; +- disk database schema and online upgrade; +- `LOAD MYSQL SERVERS TO RUNTIME`; +- `SAVE MYSQL SERVERS FROM RUNTIME`; +- `SAVE MYSQL SERVERS TO DISK`; +- ProxySQL configuration-file import and export; +- ProxySQL Cluster fetch, insert, checksum, and conflict handling; +- `dump_table_mysql("mysql_aws_aurora_hostgroups")`; +- test/bootstrap table definitions. + +ProxySQL Cluster synchronization must include the two configured values. NULL +must remain NULL on the receiving peer. + +Every configuration-bearing path must use an explicit configured-column +projection rather than `SELECT *`. In particular: + +- `SAVE MYSQL SERVERS FROM RUNTIME` copies the configured Aurora columns and + excludes `bgd_status`; +- the ProxySQL Cluster fetch/query, checksum, and insert paths include the two + green hostgroups and exclude `bgd_status`; +- disk saves and configuration-file export never persist `bgd_status`. + +BGD's `bgd_status`, transient member map, cached IPs, and pins are node-local +runtime state and are not cluster-synced as configuration. + +## 9. Schema Upgrade + +A new Aurora hostgroups schema version must add the two nullable columns while +preserving every existing row and column value. + +Existing rows are migrated with: + +```text +green_writer_hostgroup = NULL +green_reader_hostgroup = NULL +``` + +After upgrade, existing active Aurora rows follow the global variable: + +- with auto-discovery enabled, they are eligible for automatic BGD discovery; +- with auto-discovery disabled, they retain existing Aurora monitoring only. + +This is backward-compatible at the table-data level because no green +hostgroups are invented during migration. + +`bgd_status` requires no disk-schema migration because it exists only in the +runtime table. The runtime table is created with the new column, and every row +starts at `NONE` until its local Aurora worker publishes another state. + +## 10. Non-Goals of This Configuration Change + +This configuration design does not: + +- add an Aurora-specific configuration table; +- create Aurora rows in `mysql_aws_rds_bgd_hostgroups`; +- require explicit green `mysql_servers` rows in automatic mode; +- add a second per-cluster monitor worker; +- add a separate `bgd_enabled` column; +- add a runtime `mode` or `auto_generated` column; +- persist transient BGD FSM state to disk; +- alter the existing RDS Multi-AZ BGD configuration contract. + +## 11. Required Configuration Tests + +1. Both green hostgroups NULL load successfully. +2. Both green hostgroups non-NULL load successfully. +3. Mixed NULL/non-NULL values are rejected. +4. Duplicate or overlapping blue/green hostgroups are rejected. +5. Explicit mode operates with global auto-discovery disabled. +6. Automatic mode starts only when global auto-discovery is enabled. +7. Disabling auto-discovery does not abort an active automatic switchover. +8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, + and cluster synchronization round trips. +9. Online upgrade preserves existing rows and initializes both new fields to + NULL. +10. Changing either green hostgroup refreshes only the affected Aurora worker. +11. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. +12. New runtime rows initialize `bgd_status` to `NONE`. +13. Runtime `bgd_status` follows every Aurora FSM transition. +14. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes + to `NONE`. +15. Reloading an existing Aurora row preserves its `bgd_status` and active FSM + state. +16. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster + synchronization exclude `bgd_status`. +17. ProxySQL Cluster peers retain their own node-local `bgd_status` values. +18. Existing Multi-AZ BGD configuration and tests remain unchanged. + +## 12. Acceptance Criteria + +The configuration/runtime integration is complete when: + +1. Aurora BGD explicit mode is configured solely by the two green hostgroup + columns on `mysql_aws_aurora_hostgroups`. +2. Aurora BGD automatic mode is controlled by the existing + `aws_blue_green_deployment_auto_discovery` variable when both columns are + NULL. +3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve + the fields and their NULL values. +4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a + writer hostgroup. +5. No Aurora automatic-discovery row is generated in + `mysql_aws_rds_bgd_hostgroups`. +6. `runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes the local Aurora BGD + FSM state without being saved, exported, or cluster-synchronized. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md new file mode 100644 index 0000000000..abd7bccfd8 --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -0,0 +1,468 @@ +# Aurora BGD Monitor Loop and FSM Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft + +**Scope:** Aurora MySQL blue/green deployments exposed through +`mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. + +**Evidence basis:** An observed Aurora switchover captured on 2026-07-30. The +relevant timings and behavioral conclusions are summarized in Section 3 so the +specification is self-contained. + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the Aurora worker loop, topology and membership +discovery, cached member identity and IP state, DNS and pool actions, the BGD +FSM, rollback, and completion-latch behavior. Public table schemas, +persistence, and cluster synchronization belong to the configuration/runtime +specification. Simulator mechanics and the executable coverage matrix belong +to the simulator/testing specification. + +## 1. Purpose + +Provide a low-disruption ProxySQL switchover path for Aurora MySQL blue/green +deployments. ProxySQL discovers the complete target Aurora cluster before the +switchover, caches its member IPs, redirects the configured production +hostnames to those IPs during post-processing, and returns to normal DNS routing +as soon as AWS reports the target switchover complete. + +This is an Aurora-specific state machine. It does not replace the existing RDS +Multi-AZ instance behavior, where target `SWITCHOVER_COMPLETED` may describe only +the writer and reader cleanup must wait for a later topology-table drain. + +## 2. Terminology + +- **Blue:** The source/current production Aurora cluster before switchover. +- **Green:** The target Aurora cluster before promotion. +- **Canonical hostname/identifier:** The normal production instance name without + AWS's temporary `-green-` component. +- **Green hostname/identifier:** The target instance name containing + `-green-`. +- **Target cluster endpoint:** The TARGET endpoint returned by + `mysql.rds_topology`. For Aurora, this is a cluster endpoint, not an individual + writer endpoint. +- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a canonical blue + instance hostname to its cached green instance IP. +- **Probe pin:** The cached green writer/cluster IP used by the BGD monitor to + keep querying topology while canonical blue endpoints are unavailable. +- **Deployment fingerprint:** The TARGET topology identity retained after + cleanup to suppress repeated handling of the same completed row. At minimum, + it contains TARGET `id`, endpoint, and port. + +## 3. AWS/Aurora Behavioral Contract + +The Aurora implementation relies on these properties: + +1. `mysql.rds_topology` exposes SOURCE and TARGET cluster endpoints and a + monotonic status progression: + + ```text + AVAILABLE + -> SWITCHOVER_INITIATED + -> SWITCHOVER_IN_PROGRESS + -> SWITCHOVER_IN_POST_PROCESSING + -> TARGET-only SWITCHOVER_COMPLETED + ``` + +2. Querying `REPLICA_HOST_STATUS` through the target cluster endpoint returns + all current target members: exactly one writer and every Aurora reader. +3. The writer is the row whose `SESSION_ID` is `MASTER_SESSION_ID`; the other + current rows are readers. +4. During post-processing, target `SERVER_ID` values change from temporary + green identifiers to the canonical production identifiers. Reader + `SESSION_ID` values remain stable across the rename. +5. `SWITCHOVER_IN_POST_PROCESSING` is the AWS routing barrier. ProxySQL may + redirect traffic at this status without an additional target-writability + check. +6. TARGET `SWITCHOVER_COMPLETED` means canonical Aurora writer and reader DNS + cutover has completed. ProxySQL may remove all traffic pins without a + separate DNS-resolution verification. +7. The TARGET-only completed row can remain in `mysql.rds_topology` after + routing cleanup. Repeated completed rows describe the same completed + deployment and must be ignored until the table drains. + +The observed Aurora run supported these assumptions: + +- target writer writable: `T+14.438s`; +- target post-processing: `T+15.795s`; +- all target members had canonical `SERVER_ID`s: `T+21.380s`; +- canonical writer/readers accepted fresh connections: by `T+31.212s`; +- TARGET `SWITCHOVER_COMPLETED`: `T+40.655s`; +- topology table drained: `T+65.730s`. + +Thus, member cutover completed before TARGET completion; table drain was not a +reader-cutover barrier for Aurora. + +## 4. Aurora Detection + +The Aurora state machine must be selected only for an Aurora deployment. +Detection must establish both of the following: + +1. The TARGET topology endpoint is an Aurora cluster endpoint. +2. `REPLICA_HOST_STATUS` membership discovery succeeds through that endpoint. + +The existing Multi-AZ state machine remains active for non-Aurora TARGET +endpoints. + +## 5. Membership Discovery + +### 5.1 Source of truth + +`REPLICA_HOST_STATUS` is the source of truth for target Aurora membership. +Configured green hostgroups are not the source of membership and are not +required for automatic mode. + +The discovery query must include at least: + +```sql +SELECT + SERVER_ID, + SESSION_ID, + LAST_UPDATE_TIMESTAMP, + IS_CURRENT +FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS +ORDER BY SERVER_ID; +``` + +The implementation may retain the existing lag and CPU columns used by the +Aurora monitor. + +Only current rows are eligible for the active member snapshot. + +### 5.2 Constructing target instance hostnames + +The target cluster endpoint has the form: + +```text +.cluster- +``` + +A member hostname is formed as: + +```text +. +``` + +For example: + +```text +aurora-mysql-1-green-jkpanw.cluster-c1yqcg0ie39o.eu-north-1.rds.amazonaws.com + +aurora-mysql-1-writer-green-drodij.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com +``` + +### 5.3 Building blue/green pairs + +For every target member, maintain: + +```text +target SERVER_ID +SESSION_ID +role: writer or reader +green hostname +green IP +canonical blue hostname +port +configured blue server attributes +``` + +Before AWS renames the members, the canonical identifier is obtained by +removing the `-green-` component. After AWS renames them, the returned +`SERVER_ID` is already canonical. + +The writer pair is correlated by its unique writer role and cached IP. Reader +pairs are additionally correlated across renaming by their stable +`SESSION_ID`s. + +There must be exactly one writer pair. Every configured/current blue Aurora +reader must have a corresponding target reader pair before traffic pinning is +performed. + +### 5.4 Auto-generated configuration + +When `green_writer_hostgroup` and `green_reader_hostgroup` are `NULL`, the BGD +worker must still discover and map the target writer and all target readers in +memory. + +The absence of explicit green hostgroups means "no user-managed staging +hostgroups"; it does not mean "no green readers." + +The Aurora auto-generated path must not depend on green `mysql_servers` rows and +must not use the writer-only fallback as its normal reader policy. + +If the target cluster actually has no readers, the snapshot naturally contains +only the writer. If a membership query fails or yields a transient incomplete +snapshot for a cluster expected to have readers, retain the last complete +snapshot or defer the phase action. Do not reinterpret a failed/incomplete +query as a reader-less deployment. + +## 6. IP Resolution and Cached State + +During `AVAILABLE`, `SWITCHOVER_INITIATED`, and `SWITCHOVER_IN_PROGRESS`, the +worker must: + +1. Query target `REPLICA_HOST_STATUS`. +2. Build or refresh the complete member snapshot. +3. Resolve every green-suffixed member hostname. +4. Cache every member IP independently of DNS TTL expiry during the active + switchover. +5. Pin topology probing to the cached target writer/cluster IP so monitoring + survives the canonical endpoint outage and later retirement of green DNS. + +Discovery and identity refresh should continue during post-processing because +`SERVER_ID`s are renamed in that phase. Cached pre-rename IPs remain the traffic +pin values. + +Worker restart or configuration refresh must preserve enough state to avoid +losing: + +- the deployment fingerprint; +- the complete member snapshot; +- cached target IPs; +- applied traffic-pin flags; +- the probe pin; +- terminal completed/drain-wait state. + +## 7. Aurora State Machine + +### 7.1 States + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +TARGET `SWITCHOVER_COMPLETED` is an event that performs routing cleanup and +enters the internal `SWITCHOVER_COMPLETED` terminal latch; it is not a separate +reader-switchover phase. The runtime +`runtime_mysql_aws_aurora_hostgroups.bgd_status` column exposes these internal +states. + +### 7.2 State transitions and actions + +| Current observation | Required action | Polling cadence | +|---|---|---| +| No deployment | Remain `NONE`. | Normal discovery cadence | +| `AVAILABLE` | Build complete target membership, resolve all IPs, establish probe pin. | Normal/configured cadence | +| `SWITCHOVER_INITIATED` | Refresh membership/IPs, enable BGD in-progress protection. | Fast cadence | +| `SWITCHOVER_IN_PROGRESS` | Refresh membership/IPs and retain probe pin. | Fast cadence | +| First `SWITCHOVER_IN_POST_PROCESSING` | Apply all traffic pins and drain blue pools. | Fast cadence | +| Repeated `SWITCHOVER_IN_POST_PROCESSING` | Retry only unresolved/unapplied idempotent pair actions. | Fast cadence | +| First TARGET `SWITCHOVER_COMPLETED` | Remove all pins and perform immediate routing cleanup; enter the internal `SWITCHOVER_COMPLETED` latch. | Return to normal/slower cadence | +| Repeated same TARGET `SWITCHOVER_COMPLETED` while waiting | No-op. | Normal/slower cadence | +| Empty/absent table while waiting | Clear terminal fingerprint and return to `NONE`. | Normal discovery cadence | +| Different deployment fingerprint while waiting | Rearm and process the new deployment. | Appropriate new-state cadence | + +"Slower cadence" means a larger interval/lower polling frequency than the +active switchover cadence. + +## 8. Post-Processing Traffic Redirection + +On the first valid, monotonic TARGET +`SWITCHOVER_IN_POST_PROCESSING` observation: + +1. Require a complete cached writer/reader map with resolved target IPs. +2. Do **not** issue or wait for an additional writer `read_only` probe. +3. For every pair, pin: + + ```text + canonical blue hostname -> cached green IP + ``` + +4. Drain server connections for every pinned canonical blue hostname. +5. Purge free blue-host connection-pool entries so the next backend connection + uses the pinned target IP. +6. Keep mapped reader rows eligible in their existing reader hostgroup. + +Pinning, draining, and purging must be idempotent per member. A repeated +post-processing observation must not repeatedly drain an already transitioned +member. + +### 8.1 Reader policy + +The Aurora path must not shun and later unshun readers as part of a normal +switchover. All Aurora readers are discovered from `REPLICA_HOST_STATUS`, mapped, +and pinned together with the writer. + +The current writer-fallback behavior for missing green reader pairs remains a +Multi-AZ/instance policy; it is not the Aurora auto-configuration policy. + +The BGD in-progress marker must continue to suppress ordinary read-only monitor +actions that would conflict with this state machine. + +## 9. Completion and Immediate Cleanup + +On the first TARGET-only `SWITCHOVER_COMPLETED` observation, ProxySQL must trust +AWS's completion state and immediately: + +1. Remove the DNS-cache pin for every canonical writer and reader hostname. + `dns_cache->remove()` must invalidate the pinned/local entry so normal DNS + resolution resumes. +2. Remove the topology probe pin. +3. Stop fast switchover polling. +4. Clear BGD in-progress/read-only-monitor protection. +5. Drain obsolete pools belonging to explicit green hostgroups, subject to the + configured OFFLINE status preservation policy. +6. Preserve configured `mysql_servers` rows; cleanup concerns runtime routing + state and pools, not user configuration deletion. +7. Clear the active mapping/resolution state after retaining the minimal + terminal deployment fingerprint. +8. Enter the internal `SWITCHOVER_COMPLETED` latch. + +The completion path must **not**: + +- perform a new DNS-resolution verification before removing pins; +- wait for `mysql.rds_topology` to become empty; +- enter `READER_SWITCHOVER_IN_PROGRESS`; +- shun or unshun Aurora readers; +- process the same completed TARGET row more than once. + +## 10. `SWITCHOVER_COMPLETED` Terminal Latch + +The internal `SWITCHOVER_COMPLETED` state is a terminal latch, not an active +routing phase. All routing cleanup has already completed. It shares the AWS +event name intentionally but persists as ProxySQL runtime state while waiting +for topology drain. + +The worker retains only the deployment fingerprint and enough status to +recognize repeated results. While latched: + +- the same TARGET `SWITCHOVER_COMPLETED` result is ignored; +- query/connect errors do not cause rollback, repinning, or repeated cleanup; +- polling runs at the normal/slower cadence; +- successful empty results or confirmed table absence release the latch and + return the worker to `NONE`; +- a different deployment fingerprint may release/reinitialize the latch for a + new deployment. + +Table drain is therefore only the FSM rearm signal. It is not a writer or reader +availability signal for Aurora. + +## 11. Rollback Before Completion + +If AWS moves backward to `AVAILABLE`, cancels, or otherwise rolls back before +TARGET `SWITCHOVER_COMPLETED`, ProxySQL must undo only actions already applied: + +1. Remove applied traffic pins. +2. Remove the probe pin when target monitoring is no longer required. +3. Drain/purge affected canonical pools so subsequent connections follow the + restored canonical DNS path. +4. Clear BGD in-progress protection. +5. Restore normal hostgroup placement without deleting configured green rows. +6. Clear the active snapshot and rebuild it from the returned topology state. + +Rollback actions must be idempotent. Once the worker has entered the internal +`SWITCHOVER_COMPLETED` latch, transient query errors must not be treated as a +rollback. + +## 12. Separation from Multi-AZ Instance Logic + +The following existing behaviors remain valid for Multi-AZ instance BGD but +must not drive the Aurora branch: + +- treating TARGET `SWITCHOVER_COMPLETED` as writer-only completion; +- retaining reader pins until topology drains; +- inferring `READER_SWITCHOVER_IN_PROGRESS`; +- shunning unmapped blue readers; +- routing an empty reader hostgroup through the promoted writer as the normal + auto-generated fallback. + +Aurora has complete cluster membership through `REPLICA_HOST_STATUS`; its +writer and readers are switched and renamed as one cluster operation before the +TARGET completed state. + +## 13. Current Implementation Gaps + +The existing `aws_rds_bgd_build_map()` assumes the TARGET topology endpoint is +the green writer hostname and adds reader pairs only from an explicitly +configured green reader hostgroup. That assumption does not hold for Aurora: +its TARGET is a cluster endpoint. + +The Aurora implementation therefore needs a membership-driven map builder that: + +1. connects to the TARGET cluster endpoint; +2. queries `REPLICA_HOST_STATUS`; +3. constructs all member hostnames; +4. creates writer and reader pairs independently of green hostgroup + configuration; +5. retains stable reader identity across the post-processing rename; +6. applies the Aurora completion/terminal-latch FSM described above. + +The existing Multi-AZ builder and reader-cleanup FSM should remain available for +non-Aurora deployments. + +## 14. Required Tests + +### 14.1 Membership and mapping + +- Target cluster endpoint produces one writer and multiple reader pairs. +- Auto mode with NULL green hostgroups still maps all target members. +- Green-suffixed `SERVER_ID`s map to canonical blue hostnames. +- Reader `SESSION_ID`s preserve pair identity after canonical rename. +- A target cluster with no readers produces a valid writer-only snapshot. +- Failed/incomplete membership queries retain the previous complete snapshot or + defer actions. + +### 14.2 FSM + +- `AVAILABLE -> INITIATED -> IN_PROGRESS` resolves all target IPs without + changing traffic. +- First POST_PROCESSING pins and drains every pair exactly once. +- Repeated POST_PROCESSING retries only incomplete pairs. +- No additional writer-writability query gates pinning. +- Mapped readers remain eligible and are not shunned. +- First TARGET COMPLETED removes all pins and performs cleanup immediately. +- Completion does not perform a DNS verification. +- Repeated TARGET COMPLETED is a no-op while drain-wait is latched. +- Empty/absent topology releases the latch and returns to `NONE`. +- Query errors during drain-wait do not rollback or repin. +- A new deployment fingerprint can rearm discovery. + +### 14.3 Pool and DNS behavior + +- Each canonical blue hostname is pinned to the corresponding target IP. +- Pinning drains/purges old blue connections. +- Removing pins invalidates the local DNS-cache entry. +- Explicit green pools are drained at completion according to status policy. +- Configured rows are preserved. + +### 14.4 Resilience + +- Worker restart during each active phase preserves cached IPs, pins, and the + deployment fingerprint. +- Configuration refresh does not lose BGD in-progress protection. +- Rollback before completion removes all applied pins and restores normal + routing. +- Late entry at POST_PROCESSING builds a complete target snapshot before + pinning. +- Late entry at TARGET COMPLETED performs cleanup once and enters drain-wait + without first applying traffic pins. + +## 15. Acceptance Criteria + +The Aurora implementation is complete when: + +1. Auto mode discovers and maps every target Aurora member without configured + green hostgroups. +2. POST_PROCESSING redirects writer and reader traffic to cached target IPs + without reader shun/unshun actions. +3. TARGET COMPLETED removes all traffic/probe pins and completes routing cleanup + without waiting for topology drain. +4. Repeated completed rows cannot retrigger cleanup. +5. Empty/absent topology only rearms the FSM. +6. Multi-AZ instance behavior and tests remain unchanged. +7. TAP coverage verifies normal, repeated, rollback, late-entry, config-refresh, + and worker-restart paths. From 4876a87cf342283f264f491b5f384a16bfa4ebf4 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 1 Aug 2026 21:37:18 +0000 Subject: [PATCH 02/26] docs: refine Aurora BGD monitor FSM design --- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 854 +++++++++++------- 1 file changed, 514 insertions(+), 340 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index abd7bccfd8..20549dfa5c 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -4,14 +4,14 @@ **Branch:** `plan/aurora-bgd` -**Status:** Design draft +**Status:** Design approved; pending written-spec review **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** An observed Aurora switchover captured on 2026-07-30. The -relevant timings and behavioral conclusions are summarized in Section 3 so the -specification is self-contained. +**Evidence basis:** The Aurora switchover observed on 2026-07-30 and recorded in +`aurora-bgd-switchover-analysis/RESULTS-20260730T073724Z.md`. Section 3 separates +observed behavior from design policy. **Related designs:** @@ -20,50 +20,77 @@ specification is self-contained. ## Spec Boundary -This specification owns the Aurora worker loop, topology and membership -discovery, cached member identity and IP state, DNS and pool actions, the BGD -FSM, rollback, and completion-latch behavior. Public table schemas, -persistence, and cluster synchronization belong to the configuration/runtime -specification. Simulator mechanics and the executable coverage matrix belong -to the simulator/testing specification. - -## 1. Purpose - -Provide a low-disruption ProxySQL switchover path for Aurora MySQL blue/green -deployments. ProxySQL discovers the complete target Aurora cluster before the -switchover, caches its member IPs, redirects the configured production -hostnames to those IPs during post-processing, and returns to normal DNS routing -as soon as AWS reports the target switchover complete. - -This is an Aurora-specific state machine. It does not replace the existing RDS -Multi-AZ instance behavior, where target `SWITCHOVER_COMPLETED` may describe only -the writer and reader cleanup must wait for a later topology-table drain. +This specification owns the Aurora worker loop, probe ownership and cadence, +target-membership discovery, cached member identity and IP state, DNS and pool +actions, the BGD FSM, rollback, reload continuity, and completion-latch +behavior. Public table schemas, persistence, and cluster synchronization belong +to the configuration/runtime specification. Simulator mechanics and the +executable coverage matrix belong to the simulator/testing specification. + +## 1. Decision Summary + +Aurora BGD is part of the existing Aurora monitor. Each active, user-created +`mysql_aws_aurora_hostgroups` row continues to own one worker keyed by +`writer_hostgroup`. Every Aurora configuration row is user-created, and the +design starts no second BGD worker for it. + +The worker owns three logically separate probes: + +1. the existing production Aurora membership/lag probe; +2. an Aurora BGD `mysql.rds_topology` probe; +3. a target `REPLICA_HOST_STATUS` membership probe. + +The worker follows existing RDS BGD parsing, status publication, DNS pinning, +connection draining, writer-placement, rollback, reload, and cleanup patterns +where their semantics match. Aurora-specific behavior is retained where RDS +Multi-AZ instance behavior does not apply: + +- membership comes from the target Aurora cluster; +- the topology probe rotates across reachable target members instead of being + pinned to the target writer IP; +- normal Aurora probing is suspended during the active switchover phases; +- every current writer and reader must be mapped before traffic changes; +- readers are not shunned or unshunned; +- TARGET completion cleans up writer and reader routing immediately; +- topology drain is only the terminal-latch rearm signal. + +An Aurora row either has both green hostgroups configured or has neither. This +document describes the distinction only as green hostgroups configured or not +configured. The existing +`mysql-aws_blue_green_deployment_auto_discovery` variable remains only an +admission gate for starting BGD discovery when green hostgroups are not +configured, as defined by the configuration/runtime specification. ## 2. Terminology -- **Blue:** The source/current production Aurora cluster before switchover. -- **Green:** The target Aurora cluster before promotion. -- **Canonical hostname/identifier:** The normal production instance name without - AWS's temporary `-green-` component. -- **Green hostname/identifier:** The target instance name containing - `-green-`. -- **Target cluster endpoint:** The TARGET endpoint returned by - `mysql.rds_topology`. For Aurora, this is a cluster endpoint, not an individual - writer endpoint. -- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a canonical blue - instance hostname to its cached green instance IP. -- **Probe pin:** The cached green writer/cluster IP used by the BGD monitor to - keep querying topology while canonical blue endpoints are unavailable. -- **Deployment fingerprint:** The TARGET topology identity retained after - cleanup to suppress repeated handling of the same completed row. At minimum, - it contains TARGET `id`, endpoint, and port. - -## 3. AWS/Aurora Behavioral Contract - -The Aurora implementation relies on these properties: - -1. `mysql.rds_topology` exposes SOURCE and TARGET cluster endpoints and a - monotonic status progression: +- **Production cluster:** The source/current Aurora cluster before switchover + and the promoted target after completion. +- **Target cluster:** The Aurora cluster identified by the TARGET row before + promotion. +- **Canonical identifier:** The original production `SERVER_ID`, without the + temporary AWS `-green-` component. +- **Target identifier:** A pre-promotion target `SERVER_ID` containing the + temporary `-green-` component. +- **Production hostname:** The canonical instance hostname already configured + in the production writer or reader hostgroup. +- **Target hostname:** A hostname constructed from a target `SERVER_ID` and the + RDS domain suffix. +- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a production + hostname to the cached IP of its target counterpart. +- **Complete target snapshot:** Exactly one current target writer plus a unique + target counterpart for every current production member, with all target IPs + resolved. +- **Deployment fingerprint:** TARGET topology identity retained after cleanup + to recognize repeated results. At minimum it contains TARGET `id`, endpoint, + and port. + +## 3. Evidence and AWS/Aurora Behavioral Contract + +### 3.1 Directly observed behavior + +The 2026-07-30 run observed: + +1. `mysql.rds_topology` progressed through: ```text AVAILABLE @@ -73,55 +100,133 @@ The Aurora implementation relies on these properties: -> TARGET-only SWITCHOVER_COMPLETED ``` -2. Querying `REPLICA_HOST_STATUS` through the target cluster endpoint returns - all current target members: exactly one writer and every Aurora reader. -3. The writer is the row whose `SESSION_ID` is `MASTER_SESSION_ID`; the other - current rows are readers. -4. During post-processing, target `SERVER_ID` values change from temporary - green identifiers to the canonical production identifiers. Reader - `SESSION_ID` values remain stable across the rename. -5. `SWITCHOVER_IN_POST_PROCESSING` is the AWS routing barrier. ProxySQL may - redirect traffic at this status without an additional target-writability - check. -6. TARGET `SWITCHOVER_COMPLETED` means canonical Aurora writer and reader DNS - cutover has completed. ProxySQL may remove all traffic pins without a - separate DNS-resolution verification. -7. The TARGET-only completed row can remain in `mysql.rds_topology` after - routing cleanup. Repeated completed rows describe the same completed - deployment and must be ignored until the table drains. - -The observed Aurora run supported these assumptions: - -- target writer writable: `T+14.438s`; -- target post-processing: `T+15.795s`; -- all target members had canonical `SERVER_ID`s: `T+21.380s`; -- canonical writer/readers accepted fresh connections: by `T+31.212s`; -- TARGET `SWITCHOVER_COMPLETED`: `T+40.655s`; -- topology table drained: `T+65.730s`. - -Thus, member cutover completed before TARGET completion; table drain was not a -reader-cutover barrier for Aurora. - -## 4. Aurora Detection - -The Aurora state machine must be selected only for an Aurora deployment. -Detection must establish both of the following: +2. The target cluster endpoint returned exactly one writer and all current + readers through `REPLICA_HOST_STATUS`. +3. The writer row used `SESSION_ID='MASTER_SESSION_ID'`; reader rows used + distinct session identifiers. +4. During post-processing, target `SERVER_ID` values changed from temporary + green identifiers to canonical production identifiers. +5. Reader `SESSION_ID` values remained unchanged across that rename. +6. The production topology became `SWITCHOVER_IN_PROGRESS` at `T+9.864s`; the + source writer became read-only at `T+10.398s`. +7. The target writer became writable at `T+14.438s`; target post-processing was + observed at `T+15.795s`. +8. Every target member had a canonical `SERVER_ID` by `T+21.380s`. +9. TARGET completion was visible at `T+40.655s`; the topology table drained at + `T+65.730s`. + +The implementation treats these observations as the evidence for the design, +not as a formal AWS compatibility guarantee. Tests must simulate the observed +changes, and unexpected or ambiguous metadata must fail closed. + +### 3.2 Design policy derived from the observation + +- POST_PROCESSING is the routing barrier; no additional target-writability + query gates traffic pinning. +- TARGET completion means Aurora writer and reader routing cleanup can occur + immediately. +- Table drain is not a reader-availability barrier for Aurora. +- A query error, incomplete membership result, or ambiguous identity does not + advance routing actions or cause rollback. + +## 4. Worker Ownership and Probe Model + +### 4.1 Single worker owner + +The existing Aurora worker for a `writer_hostgroup` is the sole owner of normal +Aurora monitoring and the BGD FSM. It owns all per-deployment state and +serializes the three probes and their actions. A separate Aurora BGD worker must +not be started for the same row because it would duplicate queries and race on +hostgroup state. + +### 4.2 Production Aurora probe + +This is the existing random-host `REPLICA_HOST_STATUS` probe used for Aurora +role, lag, and autopurge decisions. + +- It runs normally in `NONE` and `AVAILABLE`. +- It stops issuing queries in `SWITCHOVER_INITIATED`, + `SWITCHOVER_IN_PROGRESS`, and `SWITCHOVER_IN_POST_PROCESSING`. +- It resumes after completion cleanup enters `SWITCHOVER_COMPLETED`, or + immediately after rollback. + +Suspension means the query itself is skipped. Running the query while +discarding all role, lag, and autopurge actions provides no FSM input because +the target-membership probe owns target discovery. + +### 4.3 Topology probe + +In `NONE`, the topology probe uses the existing Aurora random-host selection: +choose a random reachable production member and fall back across the remaining +members when ping or connection setup fails. + +After the TARGET endpoint has bootstrapped a complete target member list, the +topology probe applies the same selection algorithm across the current target +members. It is not pinned to the target writer and does not use a dedicated +cached-IP probe pin. When `REPLICA_HOST_STATUS` reports renamed identifiers, +the target probe-host list is refreshed to the corresponding canonical +hostnames. + +After completion cleanup, topology probing returns to random reachable +canonical production members while the terminal latch waits for table drain. + +### 4.4 Target-membership probe + +The TARGET cluster endpoint from `mysql.rds_topology` bootstraps a separate +`REPLICA_HOST_STATUS` query. Its result is parsed only into the BGD target +snapshot. It must not be passed to normal Aurora evaluation before cutover, +because doing so could prematurely move or add target members in production +hostgroups. + +The target-membership probe continues through AVAILABLE, INITIATED, IN_PROGRESS, +and POST_PROCESSING. It stops after completion cleanup or rollback has removed +the need for target-side discovery. + +### 4.5 Cadence + +| FSM state | Topology probe | Target-membership probe | Production Aurora probe | +|---|---|---|---| +| `NONE` | Configured `check_interval_ms` when BGD discovery is admitted | Off | Configured interval | +| `AVAILABLE` | Configured `check_interval_ms` | Configured interval | Configured interval | +| `SWITCHOVER_INITIATED` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_PROGRESS` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_POST_PROCESSING` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_COMPLETED` | Configured interval | Off | Configured interval | + +The 100 ms active cadence follows the existing RDS BGD worker. Query duration +is accounted for so a slow query does not create an additional full-interval +sleep. + +## 5. Discovery Bootstrap and Admission + +For an eligible row in `NONE`: + +1. Probe `information_schema.TABLES` for `mysql.rds_topology`, following the + RDS BGD table-check state. +2. When the table exists, fetch and parse `SELECT * FROM mysql.rds_topology` so + AWS column-set differences remain tolerated by the shared parser. +3. Require a structurally valid BGD result with a TARGET row, non-empty TARGET + status, endpoint, and port. +4. Publish the observed TARGET status to `bgd_status`. +5. Use the TARGET cluster endpoint to bootstrap target membership. +6. After target membership is available, rotate topology probes across target + members. -1. The TARGET topology endpoint is an Aurora cluster endpoint. -2. `REPLICA_HOST_STATUS` membership discovery succeeds through that endpoint. +A membership failure does not undo a valid observed status. It prevents any +membership-dependent routing action and is retried at the cadence for the +current phase. Missing, empty, or malformed topology does not create a new +deployment. -The existing Multi-AZ state machine remains active for non-Aurora TARGET -endpoints. +If green hostgroups are configured, BGD discovery is admitted for the +user-created Aurora row. If neither is configured, the existing global +auto-discovery variable gates admission of a new deployment. Changing that +variable after a deployment has started does not abandon the active FSM. -## 5. Membership Discovery +## 6. Target Membership and Pair Mapping -### 5.1 Source of truth +### 6.1 Query and current rows -`REPLICA_HOST_STATUS` is the source of truth for target Aurora membership. -Configured green hostgroups are not the source of membership and are not -required for automatic mode. - -The discovery query must include at least: +The target query must include at least: ```sql SELECT @@ -133,108 +238,100 @@ FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS ORDER BY SERVER_ID; ``` -The implementation may retain the existing lag and CPU columns used by the -Aurora monitor. +The implementation may retain the existing lag and CPU columns. Only rows +identified as current are eligible. The writer is the sole current row whose +`SESSION_ID` is `MASTER_SESSION_ID`; all other eligible rows are readers. -Only current rows are eligible for the active member snapshot. +### 6.2 Complete-snapshot rule -### 5.2 Constructing target instance hostnames +A new snapshot replaces the last complete snapshot only when: -The target cluster endpoint has the form: +1. it contains exactly one current target writer; +2. each current production member has exactly one target counterpart; +3. no target member maps to more than one production member; +4. no production member maps to more than one target member; +5. every target hostname has a resolved IP. -```text -.cluster- -``` +A target writer-only snapshot is valid only when the current production +snapshot also contains no readers. Query errors, empty results, multiple +writers, missing readers, duplicate identities, and unresolved IPs do not +replace the last complete snapshot. They also must not be reinterpreted as a +reader-less deployment. -A member hostname is formed as: +POST_PROCESSING actions require a complete snapshot. If none exists, the +worker holds the current status, performs no partial traffic cutover, and +retries both active probes. -```text -. -``` +### 6.3 Constructing member hostnames -For example: +The TARGET cluster endpoint has the form: ```text -aurora-mysql-1-green-jkpanw.cluster-c1yqcg0ie39o.eu-north-1.rds.amazonaws.com - -aurora-mysql-1-writer-green-drodij.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com +.cluster- ``` -### 5.3 Building blue/green pairs - -For every target member, maintain: +A target member hostname is: ```text -target SERVER_ID -SESSION_ID -role: writer or reader -green hostname -green IP -canonical blue hostname -port -configured blue server attributes +. ``` -Before AWS renames the members, the canonical identifier is obtained by -removing the `-green-` component. After AWS renames them, the returned -`SERVER_ID` is already canonical. - -The writer pair is correlated by its unique writer role and cached IP. Reader -pairs are additionally correlated across renaming by their stable -`SESSION_ID`s. +Each target member stores its role, current `SERVER_ID`, `SESSION_ID`, current +hostname, port, resolved IP, normalized canonical identifier, matching +production hostname, and per-action flags. -There must be exactly one writer pair. Every configured/current blue Aurora -reader must have a corresponding target reader pair before traffic pinning is -performed. +### 6.4 Identity across rename -### 5.4 Auto-generated configuration +Pair every target member to production by its normalized `SERVER_ID`: -When `green_writer_hostgroup` and `green_reader_hostgroup` are `NULL`, the BGD -worker must still discover and map the target writer and all target readers in -memory. +- before promotion, remove the AWS-added `-green-` component; +- after promotion, leave the already canonical identifier unchanged. -The absence of explicit green hostgroups means "no user-managed staging -hostgroups"; it does not mean "no green readers." +For readers, `SESSION_ID` is an additional continuity check. A reader whose +`SERVER_ID` changes but whose session identifier matches the cached pair is the +same target reader. A session mismatch or a conflict between normalized name +and cached session identity makes the new snapshot ambiguous. -The Aurora auto-generated path must not depend on green `mysql_servers` rows and -must not use the writer-only fallback as its normal reader policy. +For the writer, `MASTER_SESSION_ID` is only a role marker. It must not be used +as a unique instance identity. The sole writer is paired by normalized +`SERVER_ID`; no cached-IP writer identity rule is required. -If the target cluster actually has no readers, the snapshot naturally contains -only the writer. If a membership query fails or yields a transient incomplete -snapshot for a cluster expected to have readers, retain the last complete -snapshot or defer the phase action. Do not reinterpret a failed/incomplete -query as a reader-less deployment. +### 6.5 Cached IPs -## 6. IP Resolution and Cached State +During AVAILABLE and every active switchover phase, resolve all target member +hostnames and retain the last complete set of IPs independently of DNS TTL +expiry. Membership discovery continues through post-processing so renamed +identifiers refresh the host list, but pre-rename cached IPs remain valid +traffic-pin values. -During `AVAILABLE`, `SWITCHOVER_INITIATED`, and `SWITCHOVER_IN_PROGRESS`, the -worker must: +Cached target IPs are used for traffic redirection only. They do not pin the +topology probe to a particular member. -1. Query target `REPLICA_HOST_STATUS`. -2. Build or refresh the complete member snapshot. -3. Resolve every green-suffixed member hostname. -4. Cache every member IP independently of DNS TTL expiry during the active - switchover. -5. Pin topology probing to the cached target writer/cluster IP so monitoring - survives the canonical endpoint outage and later retirement of green DNS. +## 7. Worker-Owned State -Discovery and identity refresh should continue during post-processing because -`SERVER_ID`s are renamed in that phase. Cached pre-rename IPs remain the traffic -pin values. +The runtime owner for a writer hostgroup must retain: -Worker restart or configuration refresh must preserve enough state to avoid -losing: +```text +current FSM status +deployment fingerprint +last production membership snapshot +last complete target snapshot +normalized production/target member pairs +cached target IP per pair +traffic-pin-applied flag per pair +writer-demoted flag +production-probe-suspended flag +completion-cleanup-applied flag +configured green hostgroup identifiers, when present +``` -- the deployment fingerprint; -- the complete member snapshot; -- cached target IPs; -- applied traffic-pin flags; -- the probe pin; -- terminal completed/drain-wait state. +Per-member flags make repeated observations idempotent. The fingerprint keeps a +repeated completed row from starting cleanup again after the active map has +been released. -## 7. Aurora State Machine +## 8. State Machine -### 7.1 States +### 8.1 States ```text NONE @@ -245,224 +342,301 @@ SWITCHOVER_IN_POST_PROCESSING SWITCHOVER_COMPLETED ``` -TARGET `SWITCHOVER_COMPLETED` is an event that performs routing cleanup and -enters the internal `SWITCHOVER_COMPLETED` terminal latch; it is not a separate -reader-switchover phase. The runtime -`runtime_mysql_aws_aurora_hostgroups.bgd_status` column exposes these internal -states. +`runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes these worker states. +The final state is a terminal rearm latch, not a separate reader-switchover +phase. -### 7.2 State transitions and actions +### 8.2 Transition summary -| Current observation | Required action | Polling cadence | -|---|---|---| -| No deployment | Remain `NONE`. | Normal discovery cadence | -| `AVAILABLE` | Build complete target membership, resolve all IPs, establish probe pin. | Normal/configured cadence | -| `SWITCHOVER_INITIATED` | Refresh membership/IPs, enable BGD in-progress protection. | Fast cadence | -| `SWITCHOVER_IN_PROGRESS` | Refresh membership/IPs and retain probe pin. | Fast cadence | -| First `SWITCHOVER_IN_POST_PROCESSING` | Apply all traffic pins and drain blue pools. | Fast cadence | -| Repeated `SWITCHOVER_IN_POST_PROCESSING` | Retry only unresolved/unapplied idempotent pair actions. | Fast cadence | -| First TARGET `SWITCHOVER_COMPLETED` | Remove all pins and perform immediate routing cleanup; enter the internal `SWITCHOVER_COMPLETED` latch. | Return to normal/slower cadence | -| Repeated same TARGET `SWITCHOVER_COMPLETED` while waiting | No-op. | Normal/slower cadence | -| Empty/absent table while waiting | Clear terminal fingerprint and return to `NONE`. | Normal discovery cadence | -| Different deployment fingerprint while waiting | Rearm and process the new deployment. | Appropriate new-state cadence | +| Valid observation | Required transition and action | +|---|---| +| No deployment in `NONE` | Remain `NONE`; continue admitted discovery at configured cadence. | +| `AVAILABLE` | Enter/retain `AVAILABLE`; build target membership and cache IPs without changing production routing. | +| `SWITCHOVER_INITIATED` | Suspend production Aurora probing, capture rollback state, and switch BGD probes to fast cadence. | +| `SWITCHOVER_IN_PROGRESS` | Move the production writer to the reader hostgroup; do not route to target yet. | +| First `SWITCHOVER_IN_POST_PROCESSING` | With a complete snapshot, pin every production member, drain old connections, and restore the canonical writer to the writer hostgroup. | +| Repeated POST_PROCESSING | Retry only incomplete idempotent member actions. | +| First TARGET-only `SWITCHOVER_COMPLETED` | Remove all pins, perform immediate cleanup, resume normal Aurora monitoring, and enter the completed latch. | +| Repeated same completed TARGET | No-op while latched. | +| Successful empty/absent topology while latched | Release the fingerprint and return to `NONE`. | +| Earlier valid status before completion | Run rollback, then enter the earlier state. | +| Successful empty/absent topology before completion | Treat as cancellation and run rollback. | +| Query/connect error in any state | Retain state and retry; never infer rollback or table drain. | +| Different valid fingerprint while latched | Rearm and process the new deployment. | -"Slower cadence" means a larger interval/lower polling frequency than the -active switchover cadence. +## 9. Phase Actions -## 8. Post-Processing Traffic Redirection +### 9.1 `AVAILABLE` -On the first valid, monotonic TARGET -`SWITCHOVER_IN_POST_PROCESSING` observation: - -1. Require a complete cached writer/reader map with resolved target IPs. -2. Do **not** issue or wait for an additional writer `read_only` probe. -3. For every pair, pin: - - ```text - canonical blue hostname -> cached green IP - ``` +On entry and repeated observations: -4. Drain server connections for every pinned canonical blue hostname. -5. Purge free blue-host connection-pool entries so the next backend connection - uses the pinned target IP. -6. Keep mapped reader rows eligible in their existing reader hostgroup. +1. Publish `bgd_status=AVAILABLE`. +2. Run normal production Aurora monitoring unchanged. +3. Build or refresh the last complete target snapshot. +4. Resolve and retain every target IP. +5. Keep all production DNS, pools, and hostgroup placement unchanged. -Pinning, draining, and purging must be idempotent per member. A repeated -post-processing observation must not repeatedly drain an already transitioned -member. +Configured green hostgroups, when present, remain user configuration and are +not the target-membership source of truth. When they are absent, the complete +target map remains worker-local. No Aurora configuration row is generated in +either case. -### 8.1 Reader policy +### 9.2 `SWITCHOVER_INITIATED` -The Aurora path must not shun and later unshun readers as part of a normal -switchover. All Aurora readers are discovered from `REPLICA_HOST_STATUS`, mapped, -and pinned together with the writer. +On entry: -The current writer-fallback behavior for missing green reader pairs remains a -Multi-AZ/instance policy; it is not the Aurora auto-configuration policy. +1. Publish `bgd_status=SWITCHOVER_INITIATED`. +2. Capture production writer placement and other state required for rollback. +3. Suspend the normal production Aurora probe. +4. Switch topology and target-membership probes to 100 ms. +5. Continue refreshing target membership and IPs. +6. Make no traffic-routing or hostgroup-placement change. -The BGD in-progress marker must continue to suppress ordinary read-only monitor -actions that would conflict with this state machine. +### 9.3 `SWITCHOVER_IN_PROGRESS` -## 9. Completion and Immediate Cleanup +On the first valid observation: -On the first TARGET-only `SWITCHOVER_COMPLETED` observation, ProxySQL must trust -AWS's completion state and immediately: +1. Publish `bgd_status=SWITCHOVER_IN_PROGRESS`. +2. Retain fast BGD probes and the suspended production probe. +3. Move the current production writer to the reader hostgroup using the same + writer-demotion behavior as RDS BGD. +4. Record that demotion so rollback can restore it. +5. Do not redirect any production hostname to target yet. -1. Remove the DNS-cache pin for every canonical writer and reader hostname. - `dns_cache->remove()` must invalidate the pinned/local entry so normal DNS - resolution resumes. -2. Remove the topology probe pin. -3. Stop fast switchover polling. -4. Clear BGD in-progress/read-only-monitor protection. -5. Drain obsolete pools belonging to explicit green hostgroups, subject to the - configured OFFLINE status preservation policy. -6. Preserve configured `mysql_servers` rows; cleanup concerns runtime routing - state and pools, not user configuration deletion. -7. Clear the active mapping/resolution state after retaining the minimal - terminal deployment fingerprint. -8. Enter the internal `SWITCHOVER_COMPLETED` latch. +The move is idempotent. A repeated observation does not repeat a completed +hostgroup action. -The completion path must **not**: +### 9.4 `SWITCHOVER_IN_POST_PROCESSING` -- perform a new DNS-resolution verification before removing pins; -- wait for `mysql.rds_topology` to become empty; -- enter `READER_SWITCHOVER_IN_PROGRESS`; -- shun or unshun Aurora readers; -- process the same completed TARGET row more than once. +POST_PROCESSING is the only pre-completion traffic-redirection boundary. -## 10. `SWITCHOVER_COMPLETED` Terminal Latch +Before applying any action, require a complete target snapshot. Then: -The internal `SWITCHOVER_COMPLETED` state is a terminal latch, not an active -routing phase. All routing cleanup has already completed. It shares the AWS -event name intentionally but persists as ProxySQL runtime state while waiting -for topology drain. +1. Publish `bgd_status=SWITCHOVER_IN_POST_PROCESSING`. +2. Do not issue an additional target-writability probe. +3. For every writer and reader pair, install: -The worker retains only the deployment fingerprint and enough status to -recognize repeated results. While latched: - -- the same TARGET `SWITCHOVER_COMPLETED` result is ignored; -- query/connect errors do not cause rollback, repinning, or repeated cleanup; -- polling runs at the normal/slower cadence; -- successful empty results or confirmed table absence release the latch and - return the worker to `NONE`; -- a different deployment fingerprint may release/reinitialize the latch for a - new deployment. - -Table drain is therefore only the FSM rearm signal. It is not a writer or reader -availability signal for Aurora. + ```text + production hostname -> cached target IP + ``` -## 11. Rollback Before Completion +4. Reuse the RDS BGD connection-drain behavior for each production hostname: + drop free backend connections immediately and mark used connections + unhealthy/non-reusable. +5. Purge the corresponding monitor connection-pool entries so new connections + resolve through the traffic pin. +6. After the writer pin is installed, restore the canonical writer to the + writer hostgroup. Its placement in the reader hostgroup follows the existing + `writer_is_also_reader` configuration. +7. Keep canonical readers eligible in the reader hostgroup. Do not shun or + unshun them. + +Every pair records whether pinning and draining completed. Repeated +POST_PROCESSING results retry only unapplied actions and do not repeatedly drain +an already transitioned member. + +### 9.5 `SWITCHOVER_COMPLETED` -If AWS moves backward to `AVAILABLE`, cancels, or otherwise rolls back before -TARGET `SWITCHOVER_COMPLETED`, ProxySQL must undo only actions already applied: +On the first TARGET-only completed result: + +1. Remove every production-hostname DNS pin immediately. `dns_cache->remove()` + must invalidate the local pinned entry so normal DNS resolution resumes. +2. Do not wait for topology drain or perform a separate DNS verification. +3. Preserve current production connections; they already point to promoted + members. Drain only obsolete pools belonging to configured green + hostgroups, subject to the configured OFFLINE-status preservation policy. +4. Preserve every user-created configuration and `mysql_servers` row. +5. Clear the switchover guard and resume normal Aurora monitoring. +6. Move topology probing back to random reachable canonical production members + and the configured interval. +7. Release the active member map after retaining the deployment fingerprint and + the completion-cleanup flag. +8. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. + +While latched: + +- the same TARGET completed result is ignored; +- query/connect errors do not rollback, repin, or repeat cleanup; +- a successful empty result or confirmed table absence returns the worker to + `NONE`; +- a different valid deployment fingerprint rearms discovery. + +The topology drain is therefore only an FSM rearm signal. + +## 10. Rollback and Error Handling + +### 10.1 Rollback before completion + +A successful, structurally valid result for the same deployment with an +earlier status follows the existing RDS BGD backward-transition behavior. A +successful empty result or confirmed table absence before completion is treated +as cancellation. Either condition runs idempotent rollback: + +1. Remove every traffic pin that was applied. +2. Drain/purge affected production-hostname pools so subsequent connections use + restored canonical DNS. +3. Restore the production writer to the writer hostgroup when it was demoted. +4. Restore its reader placement according to `writer_is_also_reader`. +5. Resume normal production Aurora monitoring. +6. Return topology probing to production members and the appropriate cadence. +7. Preserve user-created green-hostgroup and server rows. +8. If AWS returned to AVAILABLE, rebuild target discovery in AVAILABLE; if the + topology disappeared, clear the deployment and return to `NONE`. + +Rollback does not drain configured green-hostgroup pools merely because an +attempt was cancelled. + +### 10.2 Errors and incomplete data + +- A topology or membership query/connect error retains the current state. +- An invalid or incomplete target snapshot retains the last complete snapshot. +- No error is interpreted as an empty table, reader-less cluster, cancellation, + or successful completion. +- No routing action runs without its state-specific prerequisites. +- Errors in the completed latch do not release or restart it. + +## 11. Late Entry, Reload, Removal, and Concurrency + +### 11.1 Late entry + +A worker that first observes INITIATED, IN_PROGRESS, or POST_PROCESSING rebuilds +all prerequisites before applying that phase's actions. In particular, +POST_PROCESSING cannot pin traffic until it has a complete target snapshot. + +A worker that first observes TARGET-only completion performs completion cleanup +once, even when no traffic pins were recorded, retains the fingerprint, and +enters the terminal latch. It must not manufacture or replay earlier phase +actions. + +### 11.2 Configuration and variable refresh + +An unrelated `LOAD MYSQL SERVERS TO RUNTIME` or variable refresh must preserve +the FSM status, deployment fingerprint, complete snapshots, cached IPs, applied +pin flags, writer-demotion state, probe-suspension state, and terminal latch. + +Changing configured green hostgroups refreshes the staging/pool references but +does not restart an active deployment from `NONE`. Disabling the global +auto-discovery variable prevents admission of a new no-green-hostgroup +deployment; it does not abort one already active. + +### 11.3 Row removal and worker exit + +Removing or deactivating the owning Aurora row follows the existing RDS BGD +worker-removal cleanup pattern: remove applied pins, restore safe production +writer placement where possible, release suspended monitoring state, and then +terminate the worker. No user configuration row is deleted as a side effect. -1. Remove applied traffic pins. -2. Remove the probe pin when target monitoring is no longer required. -3. Drain/purge affected canonical pools so subsequent connections follow the - restored canonical DNS path. -4. Clear BGD in-progress protection. -5. Restore normal hostgroup placement without deleting configured green rows. -6. Clear the active snapshot and rebuild it from the returned topology state. +### 11.4 Concurrent deployments -Rollback actions must be idempotent. Once the worker has entered the internal -`SWITCHOVER_COMPLETED` latch, transient query errors must not be treated as a -rollback. +All state is keyed by production `writer_hostgroup`. Probes, fingerprints, +snapshots, pins, writer placement, reloads, rollback, and completion for one +Aurora row must not mutate another row's state. -## 12. Separation from Multi-AZ Instance Logic +## 12. Separation from RDS Multi-AZ Instance Logic -The following existing behaviors remain valid for Multi-AZ instance BGD but -must not drive the Aurora branch: +The following RDS Multi-AZ instance behaviors must not drive the Aurora branch: -- treating TARGET `SWITCHOVER_COMPLETED` as writer-only completion; -- retaining reader pins until topology drains; -- inferring `READER_SWITCHOVER_IN_PROGRESS`; -- shunning unmapped blue readers; -- routing an empty reader hostgroup through the promoted writer as the normal - auto-generated fallback. +- pinning the topology probe to a single green writer IP; +- treating TARGET completion as writer-only completion; +- entering `READER_SWITCHOVER_IN_PROGRESS`; +- retaining reader pins until topology drain; +- shunning unmapped production readers; +- routing an empty reader hostgroup through the promoted writer as a missing-map + fallback. -Aurora has complete cluster membership through `REPLICA_HOST_STATUS`; its -writer and readers are switched and renamed as one cluster operation before the -TARGET completed state. +Aurora reuses the RDS machinery only where the behavior matches the contracts +in this specification. ## 13. Current Implementation Gaps -The existing `aws_rds_bgd_build_map()` assumes the TARGET topology endpoint is -the green writer hostname and adds reader pairs only from an explicitly -configured green reader hostgroup. That assumption does not hold for Aurora: -its TARGET is a cluster endpoint. +The existing RDS builder assumes the TARGET topology endpoint is an instance +writer and discovers readers from configured RDS green hostgroups. Aurora's +TARGET is a cluster endpoint and its complete membership comes from +`REPLICA_HOST_STATUS`. -The Aurora implementation therefore needs a membership-driven map builder that: +The Aurora implementation therefore requires: -1. connects to the TARGET cluster endpoint; -2. queries `REPLICA_HOST_STATUS`; -3. constructs all member hostnames; -4. creates writer and reader pairs independently of green hostgroup - configuration; -5. retains stable reader identity across the post-processing rename; -6. applies the Aurora completion/terminal-latch FSM described above. +1. three logical probes in the existing per-writer-hostgroup worker; +2. target-cluster membership parsing separate from normal Aurora evaluation; +3. normalized `SERVER_ID` pairing plus reader `SESSION_ID` continuity; +4. complete-snapshot validation and cached per-member IPs; +5. normal-probe suspension during the active switchover; +6. RDS-style writer demotion, DNS pinning, pool draining, rollback, and status + publication; +7. Aurora-specific all-member POST_PROCESSING and immediate completion cleanup; +8. completion-fingerprint retention until topology drain. -The existing Multi-AZ builder and reader-cleanup FSM should remain available for -non-Aurora deployments. +The existing RDS Multi-AZ builder and FSM remain unchanged. ## 14. Required Tests -### 14.1 Membership and mapping - -- Target cluster endpoint produces one writer and multiple reader pairs. -- Auto mode with NULL green hostgroups still maps all target members. -- Green-suffixed `SERVER_ID`s map to canonical blue hostnames. -- Reader `SESSION_ID`s preserve pair identity after canonical rename. -- A target cluster with no readers produces a valid writer-only snapshot. -- Failed/incomplete membership queries retain the previous complete snapshot or - defer actions. - -### 14.2 FSM - -- `AVAILABLE -> INITIATED -> IN_PROGRESS` resolves all target IPs without - changing traffic. -- First POST_PROCESSING pins and drains every pair exactly once. -- Repeated POST_PROCESSING retries only incomplete pairs. -- No additional writer-writability query gates pinning. -- Mapped readers remain eligible and are not shunned. -- First TARGET COMPLETED removes all pins and performs cleanup immediately. -- Completion does not perform a DNS verification. -- Repeated TARGET COMPLETED is a no-op while drain-wait is latched. +### 14.1 Probe ownership and cadence + +- One Aurora worker owns all three logical probes. +- NONE and AVAILABLE use configured cadence and continue normal Aurora probes. +- INITIATED, IN_PROGRESS, and POST_PROCESSING use 100 ms BGD probes and issue no + normal Aurora query. +- Topology selection rotates across target members and falls back when a member + is unreachable. +- No target-writer IP probe pin is installed. +- COMPLETED resumes normal probing and production-member topology selection. + +### 14.2 Membership and mapping + +- Target cluster membership yields one writer and multiple reader pairs. +- No configured green hostgroups still produces a complete in-memory map. +- Green-suffixed `SERVER_ID`s normalize to production identifiers. +- Reader `SESSION_ID`s preserve identity after canonical rename. +- The writer is paired by normalized `SERVER_ID`, not `MASTER_SESSION_ID` or IP. +- A genuine no-reader cluster accepts a writer-only snapshot. +- Empty, failed, duplicate-writer, ambiguous, unresolved, and incomplete results + retain the last complete snapshot or defer actions. + +### 14.3 FSM and routing + +- AVAILABLE discovers and resolves every member without routing changes. +- INITIATED suspends normal Aurora probing without moving servers. +- IN_PROGRESS moves the production writer to the reader hostgroup exactly once. +- POST_PROCESSING requires a complete snapshot. +- First POST_PROCESSING pins and drains every member and restores the writer. +- No target-writability query gates POST_PROCESSING. +- Readers remain eligible and are never shunned. +- Repeated POST_PROCESSING retries only unapplied work. +- First TARGET completion removes all pins and cleans up immediately. +- Completion performs no DNS verification and does not wait for table drain. +- Repeated completion is a no-op while latched. - Empty/absent topology releases the latch and returns to `NONE`. -- Query errors during drain-wait do not rollback or repin. -- A new deployment fingerprint can rearm discovery. -### 14.3 Pool and DNS behavior +### 14.4 Resilience and lifecycle -- Each canonical blue hostname is pinned to the corresponding target IP. -- Pinning drains/purges old blue connections. -- Removing pins invalidates the local DNS-cache entry. -- Explicit green pools are drained at completion according to status policy. -- Configured rows are preserved. - -### 14.4 Resilience - -- Worker restart during each active phase preserves cached IPs, pins, and the - deployment fingerprint. -- Configuration refresh does not lose BGD in-progress protection. -- Rollback before completion removes all applied pins and restores normal - routing. -- Late entry at POST_PROCESSING builds a complete target snapshot before - pinning. -- Late entry at TARGET COMPLETED performs cleanup once and enters drain-wait - without first applying traffic pins. +- Backward status and pre-completion topology disappearance run rollback. +- Query errors never run rollback or release the completed latch. +- Rollback removes pins, restores writer placement, and resumes normal probes. +- Late entry at every active state reconstructs prerequisites before acting. +- Reload preserves active and terminal state. +- Changing or removing the owning row runs safe, isolated cleanup. +- Disabling auto-discovery does not abort an admitted deployment. +- Concurrent writer hostgroups remain isolated. +- Configured green pools are drained only on successful completion, with user + rows and statuses preserved. ## 15. Acceptance Criteria -The Aurora implementation is complete when: - -1. Auto mode discovers and maps every target Aurora member without configured - green hostgroups. -2. POST_PROCESSING redirects writer and reader traffic to cached target IPs - without reader shun/unshun actions. -3. TARGET COMPLETED removes all traffic/probe pins and completes routing cleanup +The Aurora monitor/FSM design is satisfied when: + +1. One existing Aurora worker owns the three probes and all state for its + writer hostgroup. +2. Every production member is mapped from target `REPLICA_HOST_STATUS` before + traffic changes, whether or not green hostgroups are configured. +3. Normal Aurora queries stop during the three active switchover phases and + resume after completion or rollback. +4. IN_PROGRESS demotes the production writer using the RDS BGD behavior. +5. POST_PROCESSING redirects writer and reader traffic using cached target IPs, + drains old connections, and never shuns readers. +6. TARGET completion removes every traffic pin and finishes routing cleanup without waiting for topology drain. -4. Repeated completed rows cannot retrigger cleanup. -5. Empty/absent topology only rearms the FSM. -6. Multi-AZ instance behavior and tests remain unchanged. -7. TAP coverage verifies normal, repeated, rollback, late-entry, config-refresh, - and worker-restart paths. +7. Repeated results, errors, reloads, late entry, rollback, worker removal, and + concurrent deployments behave idempotently and remain isolated. +8. The terminal completed state persists only until a successful topology drain + observation rearms the FSM. +9. RDS Multi-AZ instance behavior and tests remain unchanged. From 9bf316dc1d886e9a3576020d9b426dab66d3f566 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 15:21:47 +0000 Subject: [PATCH 03/26] docs: finalize Aurora BGD simulator testing design --- ...ra-bgd-cluster-simulator-testing-design.md | 605 ++++++++++-------- 1 file changed, 340 insertions(+), 265 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index f3fda79484..6796a97945 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -1,356 +1,431 @@ -# Aurora BGD Cluster Simulator and Testing Design +# AWS Aurora Blue/Green Cluster Simulator and Testing Design **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design draft +**Status:** Design approved **Related designs:** - [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) - [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) -## Spec Boundary +## 1. Scope -This specification owns the simulator capabilities and executable test coverage -needed to validate the other two specifications. It does not redefine public -configuration or FSM behavior. When a test expectation depends on those -contracts, the corresponding sibling specification is authoritative. +This specification defines the simulator services, test controllers, endpoint +model, and executable coverage for AWS Aurora MySQL blue/green deployments. +The related designs remain authoritative for configuration, runtime status, +cluster synchronization, monitor scheduling, FSM transitions, and routing +behavior. -## 1. Purpose +The test environment drives the complete Aurora BGD lifecycle without live AWS +infrastructure and retains regression coverage for ordinary Aurora monitoring +and RDS Multi-AZ BGD. -Provide deterministic coverage for Aurora MySQL blue/green deployments without -requiring live AWS infrastructure. The test environment must be able to drive -the complete BGD observation sequence, publish target Aurora membership, -simulate member renaming, inject failures, observe monitor probes, and verify -ProxySQL runtime and routing effects. +## 2. Test Architecture -The design must preserve both existing regression suites: +The `cluster_sim_aurora` group contains two complementary test styles: -- the ordinary Aurora cluster simulator continues validating Aurora role, - membership, lag, failover, and autopurge behavior; -- the RDS Multi-AZ BGD suite continues validating its existing instance-based - mapping and reader-cleanup FSM. +- `test_cluster_sim_aurora-t` runs JSON-defined ordinary Aurora scenarios + through the standalone `cluster_simulator` controller; +- interactive TAP binaries drive Aurora BGD phases through the + `BGD_Simulator` helper. -## 2. Architecture Decision +RDS Multi-AZ BGD TAP binaries remain in `cluster_sim_rds_bgd`. A simulator +group is an execution and CI bucket; simulator capabilities come from the +ProxySQL build, SQLite-server handlers, control tables, and TAP helpers. +Only one test binary controls a given ProxySQL instance at a time, and every +binary owns setup and cleanup for its simulator state. -Aurora BGD FSM tests extend the existing interactive RDS BGD simulator used by -`cluster_sim_rds_bgd-g1`. They do not add BGD sequencing to the legacy JSON -Aurora simulator. +The build contract is: -This is the preferred design because the RDS BGD simulator already provides: +- `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation; +- `TEST_RDS_BGD` enables RDS Multi-AZ BGD simulation; +- AWS BGD topology services are available under either flag; +- Aurora replica services are available under `TEST_AURORA`; +- read-only simulation is available to Aurora BGD, RDS BGD, and focused + read-only simulator builds; +- no separate BGD simulator flag is used. -- per-backend `mysql.rds_topology` responses; -- empty, absent, and error topology modes; -- read-only controls; -- ordered probe logging; -- fixed AWS-style hostname-to-loopback mappings; -- TAP-driven, phase-by-phase transitions; -- the build and CI integration required for BGD tests. +Aurora monitor queries in simulator builds are the same production queries used +against AWS. The SQLite server identifies the simulated backend from the local +IP and port on which it accepted the monitor connection, then returns the state +assigned to that address. -Aurora adds a target-cluster membership service and Aurora-specific TAP helpers -on top of that base. Shared topology, transaction, endpoint, TLS, read-only, -probe-wait, and cleanup behavior remains common. +## 3. AWS BGD Topology Service -### Alternatives not selected +Aurora BGD and RDS Multi-AZ BGD use the same simulated +`mysql.rds_topology` service. -1. Extending the JSON Aurora simulator with a long sequence of BGD states would - overload a batch-oriented two-state model and make late entry, retry, - rollback, and concurrent timing difficult to control. -2. Creating a third standalone simulator group would duplicate the existing RDS - BGD topology service, host maps, helpers, build flavor, and CI wiring. -3. Using only unit tests would not exercise monitor threads, DNS pins, - connection-pool drains, runtime status publication, or config reloads. +### 3.1 `AWS_BGD_CONTROL` -## 3. Simulator Service Contract +One row controls topology behavior for one backend: -The `TEST_RDS_BGD` SQLite3-server flavor remains the executable backend for the -interactive suite. It is extended to recognize the Aurora membership query in -addition to the existing topology and read-only queries. +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +topology_present INTEGER NOT NULL DEFAULT 0 +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` -All simulated responses are keyed by the backend address and port on which the -SQLite3 server accepted the monitor connection. This preserves isolation when -multiple deployments or multiple target endpoints are active concurrently. +`topology_present` is restricted to zero or one. A missing control row and a +row with `topology_present=0` both represent an absent topology table. A +nonzero `error_code` returns the configured MySQL error. -### 3.1 Existing topology service +### 3.2 `AWS_BGD_TOPOLOGY` -The existing service remains authoritative for: +Successful topology results are stored as: ```text -mysql.rds_topology table presence -SOURCE and TARGET rows -deployment fingerprint fields -AWS status strings -configured topology errors -ordered table-check and metadata probe logs +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +row_order INTEGER NOT NULL +id TEXT NOT NULL +endpoint TEXT NOT NULL +topology_port INTEGER NOT NULL +role TEXT NOT NULL +status TEXT NOT NULL +PRIMARY KEY (backend_ip, backend_port, row_order) ``` -Aurora TARGET rows contain a cluster endpoint. Multi-AZ TARGET rows continue to -contain an instance endpoint. The simulator must not infer deployment type from -test configuration; production detection consumes the endpoint shape and the -membership-query result. +`row_order` makes the result deterministic. Aurora TARGET rows contain a +cluster endpoint; RDS Multi-AZ TARGET rows contain an instance endpoint. The +production monitor identifies the deployment type from the AWS metadata and +membership result rather than from simulator configuration. + +### 3.3 `AWS_BGD_PROBE_LOG` -### 3.2 Aurora membership service +Every topology table check and metadata query records: -The simulator adds per-backend control and row storage for -`INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. +```text +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL # table_check or metadata +encrypted INTEGER NOT NULL +``` -Each membership row contains at least: +`encrypted` is restricted to zero or one. The monotonic sequence supports +deterministic waits and ordering assertions. + +## 4. AWS Aurora Replica Service + +### 4.1 Membership sets + +`REPLICA_HOST_STATUS` stores each Aurora membership snapshot once: ```text -row_order -SERVER_ID -SESSION_ID -LAST_UPDATE_TIMESTAMP -IS_CURRENT -CPU -REPLICA_LAG_IN_MILLISECONDS +REPLICA_SET_ID TEXT NOT NULL +SERVER_ID VARCHAR NOT NULL +SESSION_ID VARCHAR NOT NULL +CPU REAL NOT NULL +LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL +REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL +IS_CURRENT INTEGER NOT NULL DEFAULT 1 +PRIMARY KEY (REPLICA_SET_ID, SERVER_ID) ``` -`SESSION_ID='MASTER_SESSION_ID'` identifies the writer. Other current rows are -readers. `row_order` makes response ordering deterministic while allowing tests -to prove that ProxySQL does not rely on writer-first or lexical membership -ordering. +`IS_CURRENT` is restricted to zero or one. It models the AWS column used by the +Aurora BGD membership query to exclude stale or decommissioned members. +Ordinary Aurora payloads default it to one. + +`REPLICA_SET_ID` is simulator-only and is never returned to ProxySQL. It groups +one membership snapshot independently of `DOMAIN_NAME`. The simulated table +does not contain `DOMAIN_NAME`; domain configuration remains in +`mysql_aws_aurora_hostgroups` and in the ordinary Aurora JSON input, where it +is used to construct member hostnames. -The control state supports: +Blue and green members share the same RDS domain suffix, so a BGD scenario uses +distinct replica-set identifiers for its blue and green snapshots. Multiple +backend addresses can map to the same set. A cluster with N members therefore +stores N membership rows plus lightweight backend mappings, rather than one +copy of the N rows per backend. -- a successful complete membership result; -- a successful writer-only result; -- an intentionally incomplete result; -- an empty result; -- table absence/error 1146; -- an arbitrary MySQL error code and message. +### 4.2 `AWS_AURORA_REPLICA_CONTROL` -Membership updates are atomic per supplied set of backends. A test must not -expose a partially rewritten snapshot unless it explicitly selects the -incomplete-result mode. +One row maps a backend address to its replica set and controls the query +response: -### 3.3 Probe log +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +replica_set_id TEXT NOT NULL +replica_table_present INTEGER NOT NULL DEFAULT 0 +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` + +`replica_table_present` is restricted to zero or one. Response behavior is: + +- no control row or `replica_table_present=0`: MySQL error 1146; +- nonzero `error_code`: the configured MySQL error; +- present table, no error, and matching rows: return that set; +- present table, no error, and no matching rows: return a successful empty + result. + +Writer-only and intentionally incomplete snapshots are represented directly by +the rows stored for the selected set. No response-mode column is required. +Topology and replica errors remain independent because their controls are in +separate tables. -The existing BGD probe log adds an Aurora-membership probe kind. Every topology -table check, topology metadata query, and target membership query records: +### 4.3 `AWS_AURORA_REPLICA_PROBE_LOG` + +Every intercepted production `REPLICA_HOST_STATUS` query records: ```text -monotonic sequence -accepted backend IP -accepted backend port -probe kind -TLS state +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +replica_set_id TEXT NULL +encrypted INTEGER NOT NULL ``` -Tests use the log to verify probe destination, ordering, cadence class, TLS, -probe-pin retention, and return to canonical probing. They must not use fixed -sleeps when an observable probe or runtime state can serve as the wait -condition. +The accepted address is logged even when no control mapping exists or the query +returns an error. `replica_set_id` is NULL when the backend has no mapping. + +### 4.4 Query handling + +Ordinary Aurora and Aurora BGD issue their production-shaped +`REPLICA_HOST_STATUS` queries under `TEST_AURORA`. For a recognized monitor +query, the SQLite handler: + +1. reads the accepted backend IP and port; +2. records the probe; +3. reads the matching `AWS_AURORA_REPLICA_CONTROL` row; +4. returns the configured error or rewrites the query to select the mapped + replica set; +5. preserves the production query's result columns, filtering, and ordering. -### 3.4 Error isolation +The ordinary Aurora query retains its timestamp, lag, and writer filtering. +The Aurora BGD query retains `IS_CURRENT` and `SERVER_ID` ordering. Other SQL +against the simulator tables executes normally so controllers can publish +state. -Topology and membership errors are independent. A scenario can publish valid -topology with failed membership, or valid membership with failed topology. -Clearing one error source must not silently clear the other. +## 5. State Publication -Simulator cleanup removes topology, membership, read-only, and probe-log state -in one operation so each TAP binary begins from a known baseline. +### 5.1 Ordinary Aurora controller -## 4. TAP Helper Model +Ordinary Aurora JSON files retain their current schema. For each Aurora cluster, +the standalone `cluster_simulator` controller: -Aurora-specific helpers extend, rather than fork, the RDS BGD helper model. -They provide test-facing representations for: +1. uses the JSON `DOMAIN_NAME` value as the replica-set identifier; +2. constructs each member hostname from `SERVER_ID` and `DOMAIN_NAME`; +3. reads `CLUSTER_SIM_HOST_FILE` to resolve those fixed simulator hostnames to + loopback IPs; +4. writes the membership rows with a current timestamp and `IS_CURRENT=1`; +5. maps every backend that can serve the cluster membership to the same set in + `AWS_AURORA_REPLICA_CONTROL`; +6. commits the membership replacement and complete backend mapping atomically. -- a blue Aurora cluster and its canonical instance endpoints; -- a target cluster endpoint; -- one target writer and zero or more target readers; -- stable member session identities; -- pre-rename green `SERVER_ID` values; -- post-rename canonical `SERVER_ID` values; -- explicit and automatic Aurora hostgroup configuration; -- expected runtime `bgd_status` values. +The group environment already supplies `CLUSTER_SIM_HOST_FILE`; test +infrastructure uses it to populate the ProxySQL container's host map, and the +controller uses the same file as the authoritative hostname-to-IP mapping. A +configured or discovered member missing from that map is a test-setup failure. -The helper API exposes operations equivalent to: +### 5.2 Aurora BGD controller + +Interactive Aurora BGD TAP tests publish state through `BGD_Simulator`. Each +membership publication supplies: + +- one stable replica-set identifier; +- one membership snapshot; +- every cluster endpoint or member backend that must return that snapshot. + +The helper replaces the set rows and complete backend mapping in one +transaction. Blue and green use separate identifiers. Tests do not store a +blue/green flag and do not infer environment identity from hostnames. + +The same helper publishes topology rows, topology errors, replica errors, +read-only state, and probe-log checkpoints. Non-1146 errors change only control +state, so clearing them exposes the retained data. An absent topology table +clears topology rows for the addressed backend, matching the topology service's +backend-scoped storage. An absent replica table preserves its shared membership +set because other backend mappings may still serve that set. + +### 5.3 Helper boundaries + +`BGD_Simulator` owns cross-service mechanics: SQL transactions, topology and +replica controls, read-only controls, probe-log reads and waits, endpoint +predicates, and cleanup. + +Deployment models remain engine-specific. `RDS_BGD_Cluster` and related RDS +types describe RDS instance topology. Aurora types describe cluster endpoints, +members, replica sets, rename identity, and expected runtime placement. + +### 5.4 Cleanup + +Set-scoped operations remove only the supplied replica set and backend +mappings. Full scenario cleanup clears: ```text -publish topology status -publish target membership snapshot -publish renamed membership snapshot -publish empty/absent topology -inject topology or membership error -record and wait for probes -query Aurora runtime row and bgd_status -query runtime server placement and status -query connection-pool state -open backend traffic and identify accepted simulator IP -clean up ProxySQL and simulator state +READONLY_STATUS +AWS_BGD_CONTROL +AWS_BGD_TOPOLOGY +AWS_BGD_PROBE_LOG +AWS_AURORA_REPLICA_CONTROL +AWS_AURORA_REPLICA_PROBE_LOG +REPLICA_HOST_STATUS ``` -Helper methods perform control operations only. Individual test scenarios own -the sequence of AWS observations and all expected ProxySQL outcomes. +Each TAP binary begins and ends with full cleanup. Sequential group execution +is expected, but correctness does not depend on a later test overwriting stale +state. -## 5. Endpoint and DNS Model +## 6. Endpoint Model -The `cluster_sim_rds_bgd-g1` fixed host map is extended with Aurora cluster -endpoints and member endpoints. Each hostname maps to a distinct loopback -address accepted by the same SQLite3 server. +`cluster_sim_aurora/add-hosts` contains fixed aliases for: -The map includes, per test cluster: - -- canonical blue writer and reader instance names; +- canonical blue cluster members; - the green target cluster endpoint returned by `mysql.rds_topology`; -- green-suffixed target writer and reader instance names; -- canonical post-rename member names; -- a second target deployment for repeated-switchover coverage; -- additional clusters for concurrency coverage. +- green target writer and reader members; +- canonical post-switchover member names; +- repeated deployments and concurrent test clusters. + +Independently controlled backends use distinct loopback IPs. Green and +canonical aliases may share an IP where the scenario represents AWS resource +renaming. Response selection always uses the accepted IP and port, not the +requested hostname. + +The fixed host map does not model mutable Route 53 propagation. DNS-related +tests assert cached-IP pins, accepted backend addresses, connection-pool +behavior, and explicit pin removal. + +## 7. Test Coverage + +### 7.1 Simulator contracts -Green and canonical names used for identity-renaming tests may resolve to the -same target-member loopback IP where that models AWS's post-processing rename. -Canonical blue DNS remains static in the container; DNS-cache pin behavior is -verified through accepted backend addresses and explicit pin removal, not by -claiming that the simulator reproduces mutable Route 53 propagation. +Focused tests cover: -## 6. Test Layers +- table schemas, primary keys, defaults, and value constraints; +- topology and replica routing by accepted backend address; +- several backend addresses sharing one replica set; +- independent blue and green sets with overlapping `SERVER_ID` values; +- production-query result shape, filtering, and ordering; +- atomic membership and mapping replacement; +- successful complete, writer-only, incomplete, and empty results; +- missing tables and arbitrary MySQL errors; +- independence of topology and replica errors; +- TLS state and ordered probe logging; +- `CLUSTER_SIM_HOST_FILE` parsing and missing-host failures; +- set-scoped deletion and full cleanup. -### 6.1 Schema and unit coverage +### 7.2 Configuration, runtime, and cluster synchronization -Fast tests cover deterministic contracts that do not require monitor threads: +Tests owned by the configuration design cover: -- configuration and runtime table definitions and column order; +- configuration and runtime table schemas and column order; - paired-NULL and hostgroup-conflict validation; -- SQL NULL preservation in bind/extract helpers; -- config-file import and export; -- disk schema upgrade defaults; -- runtime `bgd_status` string mapping; -- configured-column cluster query and checksum projection; +- config-file, memory, runtime, and disk round trips; +- runtime-only `bgd_status` initialization and preservation; - exclusion of `bgd_status` from SAVE and cluster synchronization; -- Aurora simulator JSON parsing of the two optional green hostgroups. - -### 6.2 Admin and cluster-sync integration coverage - -Admin/TAP tests cover: - -- memory-to-runtime and runtime-to-memory round trips; -- runtime-to-disk and config-file round trips; -- new runtime rows starting at `NONE`; -- reload preservation of `bgd_status` and active worker state; -- removal and deactivation behavior; -- peer synchronization of both green hostgroups, including NULL; -- proof that peers retain independent node-local `bgd_status` values; -- unchanged RDS Multi-AZ BGD cluster synchronization. - -### 6.3 Interactive Aurora BGD coverage - -The interactive suite covers the observable behavior of the monitor/FSM spec: - -1. `AVAILABLE` discovers exactly one writer and every reader, resolves each - member, and establishes only the probe state required by the design. -2. Automatic mode with NULL green hostgroups maps all target members without - generating green hostgroups or `mysql_servers` rows. -3. Explicit mode uses configured green hostgroups as staging pools but still - treats `REPLICA_HOST_STATUS` as membership truth. -4. Initiated and in-progress observations retain the complete snapshot and - engage BGD/read-only-monitor protection. -5. The first post-processing observation pins and drains each writer/reader pair - once; repeated observations retry only incomplete member actions. -6. Member `SERVER_ID` rename preserves reader identity through stable - `SESSION_ID` values and does not change cached target IPs. -7. The first TARGET `SWITCHOVER_COMPLETED` removes all traffic and probe pins, - performs cleanup once, and publishes `bgd_status='SWITCHOVER_COMPLETED'`. -8. Repeated completed rows are no-ops; successful topology drain changes the - runtime status to `NONE` and rearms discovery. -9. A different deployment fingerprint can rearm from the terminal latch without - inheriting stale members, pins, probes, or completion flags. - -### 6.4 Resilience and edge coverage - -Scenarios also cover: - -- membership row ordering and zero-reader target clusters; -- incomplete membership retaining the last complete snapshot; -- topology and membership query failures in every phase; -- rollback from initiated, in-progress, and post-processing observations; +- synchronization of the configured green hostgroups, including NULL; +- independent node-local runtime status on cluster peers. + +### 7.3 Interactive Aurora BGD scenarios + +Interactive TAP tests cover: + +1. `AVAILABLE` discovery of the target writer and all current readers through + a complete target membership snapshot. +2. Green hostgroups configured and green hostgroups not configured, with all + Aurora hostgroup rows created by the test. +3. Independent topology, target-membership, and ordinary Aurora probes with the + cadence and on/off behavior defined by the monitor/FSM design, including + random blue-member topology probes and random green-member target probes. +4. `SWITCHOVER_INITIATED`, `SWITCHOVER_IN_PROGRESS`, and + `SWITCHOVER_IN_POST_PROCESSING` state retention and retry behavior. +5. Source-writer read-only observation and the corresponding writer-to-reader + placement. +6. Target member `SERVER_ID` rename with stable reader `SESSION_ID` identity + and retained cached IPs. +7. Idempotent per-member pinning, placement, and pool-drain actions. +8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate cleanup, terminal + latch, and runtime `bgd_status='SWITCHOVER_COMPLETED'`. +9. Repeated completion observations, successful topology drain to `NONE`, and + rearming for a different deployment fingerprint. + +### 7.4 Resilience and lifecycle scenarios + +Coverage also includes: + +- zero-reader targets and membership row ordering; +- incomplete snapshots retaining the last complete target set; +- topology, replica, and read-only query failures in each relevant phase; +- rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; -- config refresh and hostgroup refresh during active phases; -- worker restart/respawn with preserved fingerprint, members, IPs, pins, and - terminal latch; -- disabling automatic discovery during an active automatic deployment; -- deleting or deactivating an Aurora row during a switchover; -- explicit green-pool cleanup with ONLINE, SHUNNED, OFFLINE_SOFT, and - OFFLINE_HARD members; -- TLS selection for topology and membership probes; -- multiple concurrent Aurora deployments with independent state; -- simultaneous Aurora BGD and RDS Multi-AZ BGD deployments. - -## 7. Regression Coverage - -The existing `test_cluster_sim_aurora-t` JSON payload suite remains responsible -for ordinary Aurora behavior. Its schema accepts the two new green hostgroup -fields as optional values but old payloads remain valid and preserve their -existing results. - -The complete existing `test_rds_bgd_*` suite remains unchanged in semantics. -Aurora membership support is additive to the shared simulator and must not -alter instance TARGET handling, reader shun/unshun policy, writer fallback, or -Multi-AZ completion/drain behavior. - -Regression runs must include: +- configuration and hostgroup refresh during an active deployment; +- worker restart or respawn with retained deployment state; +- deleting or deactivating an Aurora row during switchover; +- ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD staging-pool members; +- TLS selection for topology and replica probes; +- repeated and concurrent Aurora deployments with isolated state. -```text -ordinary Aurora cluster simulator group -RDS Multi-AZ BGD simulator group -new Aurora BGD TAP binaries in cluster_sim_rds_bgd-g1 -configuration/unit tests -ProxySQL Cluster synchronization tests -``` +### 7.5 Regression suites + +The complete ordinary Aurora JSON suite runs through the production query path +and retains its role, failover, lag, autodiscovery, and autopurge expectations. + +The RDS Multi-AZ BGD TAP suite retains its instance-target, reader policy, +writer fallback, completion, and topology-drain expectations while using the +shared `BGD_Simulator` and `AWS_BGD_*` services. -## 8. Determinism and Timing +## 8. Determinism and Diagnostics -Tests synchronize on observable state: runtime `bgd_status`, probe sequence, -server placement, pool counters, and successful simulator control commits. -Timeouts are derived from configured monitor intervals with bounded slack. +Tests synchronize on observable state: probe sequence, runtime `bgd_status`, +server placement, pool counters, and committed simulator controls. Timeouts are +derived from configured monitor intervals with bounded slack. -Fixed sleeps are allowed only for negative assertions where no event can be -awaited directly, and must be shorter than the overall TAP timeout. Every wait -failure reports the last runtime row, relevant server/pool state, and probes -observed since the scenario checkpoint. +Fixed sleeps are permitted only for bounded negative assertions with no event +that can be awaited directly. A failed wait reports the latest runtime row, +relevant server and pool state, and probes observed since the scenario +checkpoint. -Each scenario uses unique hostgroups or performs complete cleanup. Concurrent -tests use disjoint endpoints and deployment fingerprints. +Scenarios use disjoint hostgroups, endpoints, replica-set identifiers, and +deployment fingerprints, or perform full cleanup before reuse. ## 9. Build and CI Contract -The existing `test_rds_bgd` build remains the focused local build. The combined -cluster-simulator build continues compiling both `TEST_AURORA` and -`TEST_RDS_BGD` support. +`make testaurora` produces the focused Aurora simulator build, including the +Aurora replica, shared BGD topology, and read-only services. `make test_rds_bgd` +produces the focused RDS BGD build. The combined `testall` build enables both +families. -New interactive TAP binaries register in `cluster_sim_rds_bgd-g1`, which keeps -the existing fixed-host injection, SQLite3-server startup, no-backend-infra -model, CI matrix discovery, and log collection. No new workflow or simulator -group is required. +Ordinary Aurora and Aurora BGD binaries register in +`cluster_sim_aurora-g1`. RDS Multi-AZ BGD binaries register in +`cluster_sim_rds_bgd-g1`. Central simulator CI discovers groups and binaries +from `groups.json`, so no additional workflow, simulator group family, or CI +infrastructure is required. ## 10. Non-Goals -The simulator does not claim to validate: +The simulator does not validate: - AWS control-plane APIs or real AWS timing; - mutable Route 53 propagation; - application-level latency or packet loss; - cross-process persistence of transient FSM state; -- behavior of Aurora versions that violate the AWS contract in the monitor/FSM - specification. +- Aurora versions that violate the AWS contracts established by the live-AWS + analysis. -Live-AWS evidence remains a separate validation layer for assumptions about -status ordering, member rename, writability, and DNS completion. +Live-AWS evidence remains the validation layer for status ordering, member +rename, writability, and DNS completion. ## 11. Acceptance Criteria -The simulator/testing design is satisfied when: - -1. Tests can independently control topology and target membership per backend. -2. The full Aurora BGD FSM can be driven without live AWS infrastructure. -3. Runtime status, routing, pool, DNS-pin, probe, reload, and cluster-sync - contracts have deterministic assertions. -4. Automatic and explicit modes both cover writer and all-reader membership. -5. Rename, rollback, late-entry, error, refresh, repeated, and concurrent paths - are covered. -6. Existing ordinary Aurora and RDS Multi-AZ BGD suites retain their semantics. -7. The suite runs through existing cluster-simulator build and CI plumbing with - no new infrastructure group. +The design is satisfied when: + +1. Ordinary Aurora and Aurora BGD use production monitor queries and the same + backend-address-to-replica-set service. +2. Blue and green membership snapshots are stored once and returned from every + mapped backend. +3. The complete Aurora BGD FSM and its failure paths run deterministically + without live AWS infrastructure. +4. Runtime status, routing, pools, cached IPs, probes, refresh, and cluster-sync + contracts have observable assertions. +5. Ordinary Aurora and RDS Multi-AZ BGD regressions retain their semantics. +6. The suite uses the existing Aurora and RDS simulator CI groups without a + separate feature flag or simulator family. From 360e8eb76b8d62bce682285ea98d558c73906ecb Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 15:37:00 +0000 Subject: [PATCH 04/26] docs: approve Aurora BGD configuration and FSM designs --- ...nfiguration-runtime-cluster-sync-design.md | 102 +++++++++--------- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 4 +- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index d2c4ac3c7f..c2ba431504 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -2,9 +2,9 @@ **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design draft; configuration decisions locked +**Status:** Design approved **Scope:** Configuration and runtime integration of Aurora MySQL blue/green deployment handling with the existing `mysql_aws_aurora_hostgroups` subsystem. @@ -35,10 +35,9 @@ green_writer_hostgroup INT DEFAULT NULL green_reader_hostgroup INT DEFAULT NULL ``` -The columns provide explicit staging hostgroups when configured. When both are -NULL, the existing global variable -`mysql-aws_blue_green_deployment_auto_discovery` controls automatic Aurora BGD -discovery. +The columns provide staging hostgroups when configured. When both are NULL, the +existing global variable `mysql-aws_blue_green_deployment_auto_discovery` +controls whether the worker may start BGD discovery for the row. No separate Aurora BGD enable column is added. @@ -96,28 +95,24 @@ state is meaningful and must be preserved by every load, save, export, import, and cluster-synchronization path. The runtime table mirrors these configured fields and adds only `bgd_status`. -This design does not add an `auto_generated` column because the existing Aurora -row is the owner of both normal Aurora monitoring and BGD monitoring. It also -does not add a runtime `mode` column; effective mode remains derived from the -configured green hostgroups and the global auto-discovery variable. +Every Aurora row is user-created. There is no BGD mode or generated-row concept +in this table, so no `mode` or `auto_generated` column is present. -## 3. Configuration Modes +## 3. Green Hostgroup Configuration and BGD Admission -The effective mode for an active Aurora row is determined as follows: +An active Aurora row behaves as follows: -| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Effective behavior | +| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Behavior | |---|---:|---| -| Both non-NULL | `0` or `1` | Explicit Aurora BGD monitoring | -| Both NULL | `1` | Automatic Aurora BGD monitoring | +| Both non-NULL | `0` or `1` | BGD discovery is admitted | +| Both NULL | `1` | BGD discovery is admitted | | Both NULL | `0` | Aurora monitoring without BGD discovery | | Exactly one NULL | `0` or `1` | Invalid configuration | | Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | -### 3.1 Explicit mode +### 3.1 Green hostgroups configured -A row is in explicit mode when both green hostgroups are non-NULL. - -In explicit mode: +When both green hostgroups are non-NULL: - BGD monitoring is enabled independently of the global auto-discovery value. - The target cluster writer is associated with `green_writer_hostgroup`. @@ -128,12 +123,10 @@ In explicit mode: - The green hostgroups provide optional user-visible staging/routing pools in addition to the internal BGD member map. -### 3.2 Automatic mode - -A row is in automatic mode when both green hostgroups are NULL and -`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime. +### 3.2 Green hostgroups not configured -In automatic mode: +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime: - The existing Aurora monitor detects BGD topology for the row. - The monitor discovers the target writer and all target readers through @@ -143,8 +136,8 @@ In automatic mode: - Post-processing pinning covers the writer and every reader despite the green hostgroup columns being NULL. -Automatic mode therefore does not use the writer-only fallback or reader -shun/unshun policy from the Multi-AZ instance implementation. +This path does not use the writer-only fallback or reader shun/unshun policy +from the Multi-AZ instance implementation. ### 3.3 BGD disabled for the row @@ -153,11 +146,10 @@ When both green hostgroups are NULL and existing Aurora role and lag monitoring but does not start discovery of a new BGD deployment. -Disabling global auto-discovery while an automatically discovered switchover is -already active must not abandon that switchover. The worker completes pin -cleanup and enters topology-drain wait before disabling discovery for the row. -The variable gates the start of new automatic BGD state machines, not safe -completion of one already in progress. +Disabling global auto-discovery after a switchover has started must not abandon +that switchover. The worker completes pin cleanup and enters the terminal latch +before disabling discovery for the row. The variable gates the start of a new +BGD state machine, not safe completion of one already in progress. ## 4. Validation Rules @@ -196,7 +188,8 @@ green_reader_hostgroup = -1 when SQL NULL The existing Aurora monitor worker remains one worker per active writer hostgroup. The worker owns the BGD FSM and publishes each transition to the -runtime row's `bgd_status`. Its effective BGD mode is derived from: +runtime row's `bgd_status`. BGD admission and configured staging references are +determined from: ```text AWS_Aurora_Info.active @@ -237,13 +230,13 @@ fields; paths that write back to configuration explicitly project away Changing the green hostgroups during an active switchover must not lose cached member identities, applied DNS pins, or completion-latch state. The worker must -apply the refreshed explicit staging configuration without restarting the BGD -operation from `NONE`. +apply the refreshed staging configuration without restarting the BGD operation +from `NONE`. `LOAD MYSQL VARIABLES TO RUNTIME` makes a change to `aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. -The variable controls admission of new automatic BGD operations as described in -Section 3.3. +The variable controls admission of new BGD operations for rows without green +hostgroups, as described in Section 3.3. ## 7. Runtime Table Behavior @@ -271,11 +264,11 @@ topology result then changes it to `NONE`. Query or connection errors while latched do not reset the status or repeat cleanup. An Aurora row that is not handling a BGD deployment reports `NONE`, including -ordinary Aurora monitoring when automatic discovery is disabled. +ordinary Aurora monitoring when BGD discovery is not admitted. Examples: -Explicit mode: +Green hostgroups configured: ```sql INSERT INTO mysql_aws_aurora_hostgroups ( @@ -287,7 +280,7 @@ INSERT INTO mysql_aws_aurora_hostgroups ( ) VALUES (10, 20, 11, 21, '.cluster-example.eu-north-1.rds.amazonaws.com'); ``` -Automatic mode: +Green hostgroups not configured: ```sql INSERT INTO mysql_aws_aurora_hostgroups ( @@ -302,8 +295,8 @@ SET mysql-aws_blue_green_deployment_auto_discovery = 'true'; ``` The BGD FSM state is operational monitor state, not user configuration. -`bgd_status` provides its runtime observability without changing the -explicit/automatic mode contract defined here. +`bgd_status` provides its runtime observability without introducing a BGD mode +into the configuration contract. ## 8. Persistence and Synchronization @@ -349,7 +342,7 @@ green_reader_hostgroup = NULL After upgrade, existing active Aurora rows follow the global variable: -- with auto-discovery enabled, they are eligible for automatic BGD discovery; +- with auto-discovery enabled, they are eligible for BGD discovery; - with auto-discovery disabled, they retain existing Aurora monitoring only. This is backward-compatible at the table-data level because no green @@ -365,10 +358,10 @@ This configuration design does not: - add an Aurora-specific configuration table; - create Aurora rows in `mysql_aws_rds_bgd_hostgroups`; -- require explicit green `mysql_servers` rows in automatic mode; +- require green `mysql_servers` rows when green hostgroups are not configured; - add a second per-cluster monitor worker; - add a separate `bgd_enabled` column; -- add a runtime `mode` or `auto_generated` column; +- add a `mode` or `auto_generated` column; - persist transient BGD FSM state to disk; - alter the existing RDS Multi-AZ BGD configuration contract. @@ -378,9 +371,11 @@ This configuration design does not: 2. Both green hostgroups non-NULL load successfully. 3. Mixed NULL/non-NULL values are rejected. 4. Duplicate or overlapping blue/green hostgroups are rejected. -5. Explicit mode operates with global auto-discovery disabled. -6. Automatic mode starts only when global auto-discovery is enabled. -7. Disabling auto-discovery does not abort an active automatic switchover. +5. Configured green hostgroups admit BGD discovery with global auto-discovery + disabled. +6. A row without green hostgroups starts BGD discovery only when global + auto-discovery is enabled. +7. Disabling auto-discovery does not abort an active switchover. 8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. 9. Online upgrade preserves existing rows and initializes both new fields to @@ -402,16 +397,15 @@ This configuration design does not: The configuration/runtime integration is complete when: -1. Aurora BGD explicit mode is configured solely by the two green hostgroup - columns on `mysql_aws_aurora_hostgroups`. -2. Aurora BGD automatic mode is controlled by the existing - `aws_blue_green_deployment_auto_discovery` variable when both columns are - NULL. +1. Every Aurora configuration row is user-created, and its two green hostgroup + columns are either both configured or both NULL. +2. Configured green hostgroups admit BGD discovery independently of the global + variable; when both are NULL, the existing + `aws_blue_green_deployment_auto_discovery` variable controls admission. 3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve the fields and their NULL values. 4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a writer hostgroup. -5. No Aurora automatic-discovery row is generated in - `mysql_aws_rds_bgd_hostgroups`. +5. No Aurora row is generated in `mysql_aws_rds_bgd_hostgroups`. 6. `runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes the local Aurora BGD FSM state without being saved, exported, or cluster-synchronized. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 20549dfa5c..1977ff1da8 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -2,9 +2,9 @@ **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design approved; pending written-spec review +**Status:** Design approved **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. From f6a83fe6001820fc49a6404301a2d20a5abd0910 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 19:50:20 +0000 Subject: [PATCH 05/26] docs: decouple Aurora BGD spec from local evidence path --- .../2026-07-31-aurora-bgd-monitor-fsm-design.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 1977ff1da8..26df126da7 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -9,9 +9,8 @@ **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** The Aurora switchover observed on 2026-07-30 and recorded in -`aurora-bgd-switchover-analysis/RESULTS-20260730T073724Z.md`. Section 3 separates -observed behavior from design policy. +**Evidence basis:** A timestamped live-Aurora switchover observation captured on +2026-07-30. Section 3 separates observed behavior from design policy. **Related designs:** From 05f9c43b21b9649f7ee292d115a6d3c7a0e87fd0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 20:28:09 +0000 Subject: [PATCH 06/26] docs: address Aurora BGD review feedback --- ...ra-bgd-cluster-simulator-testing-design.md | 52 ++++++++++-- ...nfiguration-runtime-cluster-sync-design.md | 83 +++++++++++++------ ...026-07-31-aurora-bgd-monitor-fsm-design.md | 58 +++++++++++-- 3 files changed, 156 insertions(+), 37 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 6796a97945..32e8864b83 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -38,6 +38,12 @@ ProxySQL build, SQLite-server handlers, control tables, and TAP helpers. Only one test binary controls a given ProxySQL instance at a time, and every binary owns setup and cleanup for its simulator state. +The group runner enforces that ownership by starting one registered binary, +waiting for it to exit, and only then starting the next binary in sorted order. +Aurora BGD registration checks must retain that serial execution contract. If a +future runner executes binaries concurrently, each binary must receive an +isolated ProxySQL instance and simulator database. + The build contract is: - `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation; @@ -66,14 +72,28 @@ One row controls topology behavior for one backend: backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL topology_present INTEGER NOT NULL DEFAULT 0 + CHECK (topology_present IN (0, 1)) error_code INTEGER NOT NULL DEFAULT 0 error_msg TEXT NOT NULL DEFAULT '' PRIMARY KEY (backend_ip, backend_port) ``` `topology_present` is restricted to zero or one. A missing control row and a -row with `topology_present=0` both represent an absent topology table. A -nonzero `error_code` returns the configured MySQL error. +row with `topology_present=0` both represent an absent topology table. Query +precedence is evaluated independently: + +| Query | Control state | Result | +|---|---|---| +| Table check | No row or `topology_present=0` | Successful empty result; `error_code` is ignored. | +| Table check | `topology_present=1` | Successful one-row result; `error_code` is ignored. | +| Metadata | No row | MySQL error 1146. | +| Metadata | `error_code!=0` | Configured MySQL error, including the defensive `topology_present=0` combination. | +| Metadata | `error_code=0`, `topology_present=0` | MySQL error 1146. | +| Metadata | `error_code=0`, `topology_present=1` | Ordered rows, including a successful empty result. | + +The helper normally publishes only consistent combinations: error 1146 marks +the table absent, and other errors retain a present table. The defensive matrix +keeps direct control-table writes deterministic. ### 3.2 `AWS_BGD_TOPOLOGY` @@ -106,6 +126,7 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL probe_kind TEXT NOT NULL # table_check or metadata encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) ``` `encrypted` is restricted to zero or one. The monotonic sequence supports @@ -125,6 +146,7 @@ CPU REAL NOT NULL LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL IS_CURRENT INTEGER NOT NULL DEFAULT 1 + CHECK (IS_CURRENT IN (0, 1)) PRIMARY KEY (REPLICA_SET_ID, SERVER_ID) ``` @@ -154,15 +176,18 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL replica_set_id TEXT NOT NULL replica_table_present INTEGER NOT NULL DEFAULT 0 + CHECK (replica_table_present IN (0, 1)) error_code INTEGER NOT NULL DEFAULT 0 error_msg TEXT NOT NULL DEFAULT '' PRIMARY KEY (backend_ip, backend_port) ``` -`replica_table_present` is restricted to zero or one. Response behavior is: +`replica_table_present` is restricted to zero or one. Absence takes precedence +over a configured error. Response behavior is: -- no control row or `replica_table_present=0`: MySQL error 1146; -- nonzero `error_code`: the configured MySQL error; +- no control row or `replica_table_present=0`: MySQL error 1146, regardless of + `error_code`; +- present table and nonzero `error_code`: the configured MySQL error; - present table, no error, and matching rows: return that set; - present table, no error, and no matching rows: return a successful empty result. @@ -182,6 +207,7 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL replica_set_id TEXT NULL encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) ``` The accepted address is logged even when no control mapping exists or the query @@ -308,6 +334,8 @@ Focused tests cover: - atomic membership and mapping replacement; - successful complete, writer-only, incomplete, and empty results; - missing tables and arbitrary MySQL errors; +- table-present/error precedence matrices for topology table-check, topology + metadata, and replica membership queries; - independence of topology and replica errors; - TLS state and ordered probe logging; - `CLUSTER_SIM_HOST_FILE` parsing and missing-host failures; @@ -358,7 +386,7 @@ Coverage also includes: - rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; - configuration and hostgroup refresh during an active deployment; -- worker restart or respawn with retained deployment state; +- in-place worker configuration refresh with retained deployment state; - deleting or deactivating an Aurora row during switchover; - ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD staging-pool members; - TLS selection for topology and replica probes; @@ -384,6 +412,13 @@ that can be awaited directly. A failed wait reports the latest runtime row, relevant server and pool state, and probes observed since the scenario checkpoint. +Random probe-selection tests do not assert one exact random order. They assert +that every observed destination belongs to the scenario's eligible set and +that fallback reaches another eligible member within a bounded number of +attempts. Failure diagnostics include the complete probe sequence since the +scenario checkpoint, so production randomness cannot make the expected result +order-dependent. + Scenarios use disjoint hostgroups, endpoints, replica-set identifiers, and deployment fingerprints, or perform full cleanup before reuse. @@ -400,6 +435,11 @@ Ordinary Aurora and Aurora BGD binaries register in from `groups.json`, so no additional workflow, simulator group family, or CI infrastructure is required. +CI must continue to invoke the group through the serial group runner described +in Section 2. Registration coverage verifies that every Aurora BGD binary is in +that group and is not scheduled simultaneously against the same ProxySQL +instance. + ## 10. Non-Goals The simulator does not validate: diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index c2ba431504..5ef200ec6b 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -102,7 +102,7 @@ in this table, so no `mode` or `auto_generated` column is present. An active Aurora row behaves as follows: -| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Behavior | +| Green hostgroup columns | `mysql-aws_blue_green_deployment_auto_discovery` | Behavior | |---|---:|---| | Both non-NULL | `0` or `1` | BGD discovery is admitted | | Both NULL | `1` | BGD discovery is admitted | @@ -110,6 +110,13 @@ An active Aurora row behaves as follows: | Exactly one NULL | `0` or `1` | Invalid configuration | | Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | +Changing `active` from `1` to `0` uses the worker-removal cleanup contract in +the monitor/FSM specification before the worker exits: remove every applied +pin, drain and purge affected production-hostname pools, restore safe writer +placement, release suspended monitoring, clear any terminal latch and retained +fingerprint, and publish `bgd_status=NONE`. The configuration and +`mysql_servers` rows remain user-owned and are not deleted by this cleanup. + ### 3.1 Green hostgroups configured When both green hostgroups are non-NULL: @@ -170,11 +177,18 @@ For each `mysql_aws_aurora_hostgroups` row: green role in another active Aurora row. 4. Existing writer-hostgroup primary-key and reader-hostgroup uniqueness rules remain in effect. -5. Invalid rows must be rejected or excluded from runtime loading with a clear - admin error identifying the writer hostgroup and conflicting fields. - -Validation must occur before publishing the new Aurora monitor resultset so a -bad row cannot partially reconfigure a running monitor worker. +5. Invalid rows use per-row isolation: each invalid row is excluded from runtime + loading, while the valid rows from the same LOAD remain eligible to publish. + The admin error identifies the rejected writer hostgroup, the conflicting + fields and values, and the other writer hostgroup involved in a cross-row + conflict. + +Validation of the complete candidate result set must finish before publishing +the filtered valid result set. Valid rows publish together only after that +validation succeeds and atomically replace the prior runtime configuration. A +rejected row contributes no new fields; if it previously owned a runtime +worker, its absence from the replacement triggers the worker-removal cleanup +contract rather than applying a mixture of old and invalid fields. ## 5. Runtime Ownership @@ -207,6 +221,13 @@ configured fields plus `bgd_status`. Its runtime dump includes all of those fields; paths that write back to configuration explicitly project away `bgd_status`. +Configuration materialization and worker status publication use the same +Hostgroups Manager write-locked update path. Reload merges configured fields +into an existing writer-hostgroup row without writing its `bgd_status`; the +worker status API updates only `bgd_status` under that lock. A reload that began +from an older snapshot therefore cannot overwrite a transition published while +the reload is being applied. + ## 6. LOAD Behavior `LOAD MYSQL SERVERS TO RUNTIME` must: @@ -216,9 +237,10 @@ fields; paths that write back to configuration explicitly project away hostgroup `0`. 3. Validate paired NULL/non-NULL and hostgroup-conflict rules. 4. Update or create the `AWS_Aurora_Info` entry. -5. Include both fields in the Aurora monitor resultset checksum. -6. Restart/refresh only the affected Aurora writer-hostgroup worker when either - green hostgroup changes. +5. Include both fields in the Aurora monitor result set checksum. +6. Refresh the affected Aurora writer-hostgroup worker in place when either + green hostgroup changes. Apply only configuration-derived fields and staging + references; do not replace the worker-owned FSM object. 7. Preserve an active BGD FSM safely across an unrelated configuration refresh. 8. Initialize `bgd_status` to `NONE` for a newly published runtime row. 9. Preserve the existing `bgd_status` when merging an existing @@ -234,7 +256,8 @@ apply the refreshed staging configuration without restarting the BGD operation from `NONE`. `LOAD MYSQL VARIABLES TO RUNTIME` makes a change to -`aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. +`mysql-aws_blue_green_deployment_auto_discovery` visible to Aurora monitor +workers. The variable controls admission of new BGD operations for rows without green hostgroups, as described in Section 3.3. @@ -371,27 +394,37 @@ This configuration design does not: 2. Both green hostgroups non-NULL load successfully. 3. Mixed NULL/non-NULL values are rejected. 4. Duplicate or overlapping blue/green hostgroups are rejected. -5. Configured green hostgroups admit BGD discovery with global auto-discovery +5. A LOAD containing valid and invalid rows excludes each invalid row, reports + its writer hostgroup and conflicting fields, and atomically publishes the + filtered valid result set. A previously active rejected row follows the safe + worker-removal cleanup path rather than receiving a partial update. +6. Configured green hostgroups admit BGD discovery with global auto-discovery disabled. -6. A row without green hostgroups starts BGD discovery only when global +7. A row without green hostgroups starts BGD discovery only when global auto-discovery is enabled. -7. Disabling auto-discovery does not abort an active switchover. -8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, +8. Disabling auto-discovery does not abort an active switchover. +9. Changing an active row to `active=0` removes pins, restores safe placement, + clears the latch and fingerprint, publishes `NONE`, and stops the worker + without deleting user configuration. +10. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. -9. Online upgrade preserves existing rows and initializes both new fields to +11. Online upgrade preserves existing rows and initializes both new fields to NULL. -10. Changing either green hostgroup refreshes only the affected Aurora worker. -11. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. -12. New runtime rows initialize `bgd_status` to `NONE`. -13. Runtime `bgd_status` follows every Aurora FSM transition. -14. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes +12. Changing either green hostgroup refreshes only the affected Aurora worker + in place. +13. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. +14. A LOAD concurrent with an FSM transition cannot overwrite the newer + `bgd_status`. +15. New runtime rows initialize `bgd_status` to `NONE`. +16. Runtime `bgd_status` follows every Aurora FSM transition. +17. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes to `NONE`. -15. Reloading an existing Aurora row preserves its `bgd_status` and active FSM +18. Reloading an existing Aurora row preserves its `bgd_status` and active FSM state. -16. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster +19. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster synchronization exclude `bgd_status`. -17. ProxySQL Cluster peers retain their own node-local `bgd_status` values. -18. Existing Multi-AZ BGD configuration and tests remain unchanged. +20. ProxySQL Cluster peers retain their own node-local `bgd_status` values. +21. Existing Multi-AZ BGD configuration and tests remain unchanged. ## 12. Acceptance Criteria @@ -401,7 +434,7 @@ The configuration/runtime integration is complete when: columns are either both configured or both NULL. 2. Configured green hostgroups admit BGD discovery independently of the global variable; when both are NULL, the existing - `aws_blue_green_deployment_auto_discovery` variable controls admission. + `mysql-aws_blue_green_deployment_auto_discovery` variable controls admission. 3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve the fields and their NULL values. 4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 26df126da7..95e4a6771f 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -204,10 +204,13 @@ For an eligible row in `NONE`: RDS BGD table-check state. 2. When the table exists, fetch and parse `SELECT * FROM mysql.rds_topology` so AWS column-set differences remain tolerated by the shared parser. -3. Require a structurally valid BGD result with a TARGET row, non-empty TARGET - status, endpoint, and port. -4. Publish the observed TARGET status to `bgd_status`. -5. Use the TARGET cluster endpoint to bootstrap target membership. +3. Require a structurally valid BGD result with a TARGET row and non-empty + TARGET `id`, status, endpoint, and port. Construct the deployment fingerprint + from at least that validated `id`, endpoint, and port. +4. Map the observed TARGET status to an Aurora FSM state, publish it to + `bgd_status`, and leave `NONE` before starting membership discovery. +5. Use the TARGET cluster endpoint to bootstrap target membership after that + state transition. 6. After target membership is available, rotate topology probes across target members. @@ -216,6 +219,23 @@ membership-dependent routing action and is retried at the cadence for the current phase. Missing, empty, or malformed topology does not create a new deployment. +Raw topology statuses map to Aurora runtime states as follows: + +| Raw TARGET status | Aurora FSM and `bgd_status` value | +|---|---| +| `AVAILABLE` | `AVAILABLE` | +| `SWITCHOVER_INITIATED` | `SWITCHOVER_INITIATED` | +| `SWITCHOVER_IN_PROGRESS` | `SWITCHOVER_IN_PROGRESS` | +| `SWITCHOVER_IN_POST_PROCESSING` | `SWITCHOVER_IN_POST_PROCESSING` | +| `SWITCHOVER_COMPLETED` in a valid TARGET-only result | `SWITCHOVER_COMPLETED` terminal latch | + +`NONE` is an internal baseline and rearm state, not an arbitrary raw status. +An unsupported or unknown raw status is invalid metadata: a worker already in +an active or terminal state retains that state, and a worker in `NONE` remains +there. The value is never copied verbatim into `bgd_status`. Aurora does not use +the RDS Multi-AZ inferred `READER_SWITCHOVER_IN_PROGRESS` state; TARGET +completion maps directly to the Aurora terminal latch after cleanup. + If green hostgroups are configured, BGD discovery is admitted for the user-created Aurora row. If neither is configured, the existing global auto-discovery variable gates admission of a new deployment. Changing that @@ -345,7 +365,30 @@ SWITCHOVER_COMPLETED The final state is a terminal rearm latch, not a separate reader-switchover phase. -### 8.2 Transition summary +### 8.2 Topology row-shape predicates + +The worker validates the complete topology result before publishing a new +status or running an FSM action: + +- An active deployment result contains exactly one SOURCE row and one TARGET + row. Both rows have non-empty `id`, endpoint, role, status, and a valid port; + both expose the same supported pre-completion status. +- A completion result contains exactly one TARGET row, has the validated + deployment-fingerprint fields, and reports `SWITCHOVER_COMPLETED`. +- A successful empty result or confirmed table absence is the explicit + cancellation or terminal-drain observation described below. +- SOURCE-only results, TARGET-only pre-completion results, duplicate SOURCE or + TARGET rows, extra or unknown roles, mismatched statuses, missing required + fields, and unsupported statuses are invalid or incomplete observations. + +Invalid or incomplete observations do not publish a new status, trigger +cleanup, release pins, or resume suspended monitoring. A worker in `NONE` +remains there; an active or latched worker retains its existing state and +retries. This validation is distinct from a successful empty or absent +topology, which has the explicit state-dependent meaning in the transition +table. + +### 8.3 Transition summary | Valid observation | Required transition and action | |---|---| @@ -590,6 +633,9 @@ The existing RDS Multi-AZ builder and FSM remain unchanged. - A genuine no-reader cluster accepts a writer-only snapshot. - Empty, failed, duplicate-writer, ambiguous, unresolved, and incomplete results retain the last complete snapshot or defer actions. +- SOURCE-only, duplicate-role, mismatched-status, unknown-status, and + TARGET-only pre-completion topology results fail closed without publishing or + transitioning; only TARGET-only completion is accepted. ### 14.3 FSM and routing @@ -626,7 +672,7 @@ The Aurora monitor/FSM design is satisfied when: 1. One existing Aurora worker owns the three probes and all state for its writer hostgroup. 2. Every production member is mapped from target `REPLICA_HOST_STATUS` before - traffic changes, whether or not green hostgroups are configured. + traffic changes, regardless of whether green hostgroups are configured. 3. Normal Aurora queries stop during the three active switchover phases and resume after completion or rollback. 4. IN_PROGRESS demotes the production writer using the RDS BGD behavior. From ea8fb4681c9e1346ae7b2690b2eb8c4eb7ccf05c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 04:49:11 +0000 Subject: [PATCH 07/26] docs: record Aurora BGD status contract --- ...ra-bgd-cluster-simulator-testing-design.md | 9 ++-- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 42 +++++++++++++++---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 32e8864b83..68dfe5ac9a 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -448,11 +448,12 @@ The simulator does not validate: - mutable Route 53 propagation; - application-level latency or packet loss; - cross-process persistence of transient FSM state; -- Aurora versions that violate the AWS contracts established by the live-AWS - analysis. +- Aurora versions that violate the AWS-provided topology-status contract. -Live-AWS evidence remains the validation layer for status ordering, member -rename, writability, and DNS completion. +The AWS-provided topology metadata semantics define the status-to-routing +contract. Live-AWS evidence remains the validation layer for observed status +ordering, member rename, DNS completion, and corroboration of the writability +transition. ## 11. Acceptance Criteria diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 95e4a6771f..a87f3ba463 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -9,8 +9,12 @@ **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** A timestamped live-Aurora switchover observation captured on -2026-07-30. Section 3 separates observed behavior from design policy. +**Evidence basis:** The AWS-team-provided *RDS Topology metadata – Overview* +document defines the routing semantics of the switchover statuses. A +timestamped live-Aurora switchover observation captured on 2026-07-30 +corroborates those semantics and supplies the observed Aurora-specific details. +Section 3 separates the supplied status contract, observed behavior, and design +policy. **Related designs:** @@ -85,7 +89,24 @@ configured, as defined by the configuration/runtime specification. ## 3. Evidence and AWS/Aurora Behavioral Contract -### 3.1 Directly observed behavior +### 3.1 AWS-provided status semantics + +The AWS-team-provided *RDS Topology metadata – Overview* document defines these +traffic-routing semantics for `mysql.rds_topology` during switchover: + +| Status | AWS-provided meaning | Write traffic | Read traffic | +|---|---|---|---| +| `SWITCHOVER_INITIATED` | Switchover was triggered, but no modifications have occurred and rollback remains possible. | Source | Source | +| `SWITCHOVER_IN_PROGRESS` | Source writes are disabled while target replication catches up; rollback remains possible. | Nowhere | Source | +| `SWITCHOVER_IN_POST_PROCESSING` | The target is promoted and can receive writes; rollback is no longer possible. | Target | Target | +| `SWITCHOVER_COMPLETED` | DNS propagation is complete and the original source endpoint points to the target. | Target | Target | + +The implementation treats these meanings as the AWS-supplied behavioral +contract for routing decisions. In particular, POST_PROCESSING definitively +means that the promoted target can accept write traffic; target readiness is +not inferred solely from one observed run. + +### 3.2 Directly observed behavior The 2026-07-30 run observed: @@ -114,14 +135,17 @@ The 2026-07-30 run observed: 9. TARGET completion was visible at `T+40.655s`; the topology table drained at `T+65.730s`. -The implementation treats these observations as the evidence for the design, -not as a formal AWS compatibility guarantee. Tests must simulate the observed -changes, and unexpected or ambiguous metadata must fail closed. +The run corroborates the AWS-provided status semantics. Details learned only +from the observation, including timings, target-member rename behavior, and +table-drain timing, are implementation evidence rather than a formal AWS +compatibility guarantee. Tests must simulate the observed changes, and +unexpected or ambiguous metadata must fail closed. -### 3.2 Design policy derived from the observation +### 3.3 Design policy derived from the contract and observation -- POST_PROCESSING is the routing barrier; no additional target-writability - query gates traffic pinning. +- POST_PROCESSING is the routing barrier because the AWS-provided semantics say + that the promoted target can receive writes at that status; no additional + target-writability query gates traffic pinning. - TARGET completion means Aurora writer and reader routing cleanup can occur immediately. - Table drain is not a reader-availability barrier for Aurora. From 529295fd8695af850764f6129a4c338910100cae Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 08:34:07 +0000 Subject: [PATCH 08/26] docs: clarify Aurora BGD switchover lifecycle --- ...ra-bgd-cluster-simulator-testing-design.md | 14 ++- ...nfiguration-runtime-cluster-sync-design.md | 8 +- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 101 +++++++++++++----- 3 files changed, 87 insertions(+), 36 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 68dfe5ac9a..95e01ddba6 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -370,9 +370,11 @@ Interactive TAP tests cover: placement. 6. Target member `SERVER_ID` rename with stable reader `SESSION_ID` identity and retained cached IPs. -7. Idempotent per-member pinning, placement, and pool-drain actions. -8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate cleanup, terminal - latch, and runtime `bgd_status='SWITCHOVER_COMPLETED'`. +7. Idempotent per-member pinning, placement, and connection-retirement marking, + without treating asynchronous physical closure as an FSM prerequisite. +8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate effect-driven + cleanup, terminal latch, and runtime + `bgd_status='SWITCHOVER_COMPLETED'`. 9. Repeated completion observations, successful topology drain to `NONE`, and rearming for a different deployment fingerprint. @@ -385,6 +387,12 @@ Coverage also includes: - topology, replica, and read-only query failures in each relevant phase; - rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; +- direct completion with an empty member map producing no production-member + routing or retirement actions; +- completion directly after in-progress restoring the writer without replaying + post-processing; +- completion after post-processing removing pins without reapplying connection + retirement or waiting for marked connections to close; - configuration and hostgroup refresh during an active deployment; - in-place worker configuration refresh with retained deployment state; - deleting or deactivating an Aurora row during switchover; diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index 5ef200ec6b..fa1b7234bd 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -327,7 +327,7 @@ Both green hostgroup columns must be supported by: - admin-memory table creation; - runtime table creation; -- disk database schema and online upgrade; +- disk database schema and in-place migration on ProxySQL startup; - `LOAD MYSQL SERVERS TO RUNTIME`; - `SAVE MYSQL SERVERS FROM RUNTIME`; - `SAVE MYSQL SERVERS TO DISK`; @@ -351,7 +351,7 @@ projection rather than `SELECT *`. In particular: BGD's `bgd_status`, transient member map, cached IPs, and pins are node-local runtime state and are not cluster-synced as configuration. -## 9. Schema Upgrade +## 9. Disk Schema Migration A new Aurora hostgroups schema version must add the two nullable columns while preserving every existing row and column value. @@ -408,8 +408,8 @@ This configuration design does not: without deleting user configuration. 10. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. -11. Online upgrade preserves existing rows and initializes both new fields to - NULL. +11. In-place disk schema migration on ProxySQL startup preserves existing rows + and initializes both new fields to NULL. 12. Changing either green hostgroup refreshes only the affected Aurora worker in place. 13. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index a87f3ba463..aa9354a4cb 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -83,6 +83,9 @@ configured, as defined by the configuration/runtime specification. - **Complete target snapshot:** Exactly one current target writer plus a unique target counterpart for every current production member, with all target IPs resolved. +- **Effect-driven cleanup:** The idempotent RDS BGD pattern that reconciles + placement and removes transient routing state from the worker's existing + member map. A member-scoped operation is a no-op when its map input is absent. - **Deployment fingerprint:** TARGET topology identity retained after cleanup to recognize repeated results. At minimum it contains TARGET `id`, endpoint, and port. @@ -106,6 +109,13 @@ contract for routing decisions. In particular, POST_PROCESSING definitively means that the promoted target can accept write traffic; target readiness is not inferred solely from one observed run. +[AWS's Aurora switchover documentation](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/blue-green-deployments-switching.html) +also states that a DB cluster included in a switchover cannot be modified while +the switchover is running. The production and target member sets are therefore +fixed from INITIATED until the first COMPLETED observation. Membership can +change while the deployment is AVAILABLE, when the normal production probe +and target-membership probe remain active and refresh their snapshots. + ### 3.2 Directly observed behavior The 2026-07-30 run observed: @@ -146,6 +156,9 @@ unexpected or ambiguous metadata must fail closed. - POST_PROCESSING is the routing barrier because the AWS-provided semantics say that the promoted target can receive writes at that status; no additional target-writability query gates traffic pinning. +- Normal monitoring refreshes membership while the deployment is AVAILABLE. + INITIATED freezes the last complete production snapshot for the duration of + the switchover, matching AWS's cluster-modification restriction. - TARGET completion means Aurora writer and reader routing cleanup can occur immediately. - Table drain is not a reader-availability barrier for Aurora. @@ -301,6 +314,13 @@ writers, missing readers, duplicate identities, and unresolved IPs do not replace the last complete snapshot. They also must not be reinterpreted as a reader-less deployment. +The production snapshot is refreshed by normal Aurora monitoring throughout +AVAILABLE. When the worker first accepts INITIATED, that last complete snapshot +becomes the fixed production membership for the switchover. The normal +production probe remains suspended while AWS prevents changes to either +included cluster; the frozen snapshot and continuously refreshed target +snapshot therefore require no separate member-set generation. + POST_PROCESSING actions require a complete snapshot. If none exists, the worker holds the current status, performs no partial traffic cutover, and retries both active probes. @@ -362,9 +382,7 @@ last complete target snapshot normalized production/target member pairs cached target IP per pair traffic-pin-applied flag per pair -writer-demoted flag production-probe-suspended flag -completion-cleanup-applied flag configured green hostgroup identifiers, when present ``` @@ -422,7 +440,7 @@ table. | `SWITCHOVER_IN_PROGRESS` | Move the production writer to the reader hostgroup; do not route to target yet. | | First `SWITCHOVER_IN_POST_PROCESSING` | With a complete snapshot, pin every production member, drain old connections, and restore the canonical writer to the writer hostgroup. | | Repeated POST_PROCESSING | Retry only incomplete idempotent member actions. | -| First TARGET-only `SWITCHOVER_COMPLETED` | Remove all pins, perform immediate cleanup, resume normal Aurora monitoring, and enter the completed latch. | +| First TARGET-only `SWITCHOVER_COMPLETED` | Run immediate effect-driven cleanup, resume normal Aurora monitoring, and enter the completed latch. | | Repeated same completed TARGET | No-op while latched. | | Successful empty/absent topology while latched | Release the fingerprint and return to `NONE`. | | Earlier valid status before completion | Run rollback, then enter the earlier state. | @@ -466,7 +484,8 @@ On the first valid observation: 2. Retain fast BGD probes and the suspended production probe. 3. Move the current production writer to the reader hostgroup using the same writer-demotion behavior as RDS BGD. -4. Record that demotion so rollback can restore it. +4. Retain the member map for writer-placement reconciliation during completion + or rollback. 5. Do not redirect any production hostname to target yet. The move is idempotent. A repeated observation does not repeat a completed @@ -497,27 +516,41 @@ Before applying any action, require a complete target snapshot. Then: 7. Keep canonical readers eligible in the reader hostgroup. Do not shun or unshun them. -Every pair records whether pinning and draining completed. Repeated -POST_PROCESSING results retry only unapplied actions and do not repeatedly drain -an already transitioned member. +Every pair records that its pin-and-retirement action was applied after the DNS +pin, free-connection deletion, used-connection unhealthy/non-reusable marking, +and monitor-pool purge calls return. This flag does not mean that every used +connection has physically closed: destruction happens asynchronously when the +connection is released and is not a completion prerequisite. Repeated +POST_PROCESSING results retry only unapplied actions and do not reapply +retirement to an already transitioned member. ### 9.5 `SWITCHOVER_COMPLETED` -On the first TARGET-only completed result: - -1. Remove every production-hostname DNS pin immediately. `dns_cache->remove()` - must invalidate the local pinned entry so normal DNS resolution resumes. -2. Do not wait for topology drain or perform a separate DNS verification. -3. Preserve current production connections; they already point to promoted - members. Drain only obsolete pools belonging to configured green - hostgroups, subject to the configured OFFLINE-status preservation policy. -4. Preserve every user-created configuration and `mysql_servers` row. -5. Clear the switchover guard and resume normal Aurora monitoring. -6. Move topology probing back to random reachable canonical production members +On the first TARGET-only completed result, run effect-driven cleanup +immediately, before publishing the terminal latch: + +1. If the existing member map contains a writer, reconcile that writer into + the writer hostgroup and apply its reader placement according to + `writer_is_also_reader`. This restores a writer moved during IN_PROGRESS when + POST_PROCESSING was not observed, and is a no-op when the writer is already + restored or the map is empty. +2. For every mapped pair, remove the production-hostname DNS entry and purge + the corresponding monitor-pool entries so normal DNS resolution resumes. + Removing an absent pin or pool entry is a no-op. +3. Do not reapply connection retirement, wait for marked connections to close, + wait for topology drain, or perform a separate DNS verification. +4. Drain only obsolete pools belonging to configured green hostgroups, subject + to the configured OFFLINE-status preservation policy. +5. Preserve every user-created configuration and `mysql_servers` row. +6. Clear any installed switchover guard and resume normal Aurora monitoring. +7. Move topology probing back to random reachable canonical production members and the configured interval. -7. Release the active member map after retaining the deployment fingerprint and - the completion-cleanup flag. -8. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. +8. Release the active member map after retaining the deployment fingerprint. +9. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. + +The cleanup path is the same regardless of the prior phase. It acts only on the +available member map and worker state, so absent inputs naturally produce +no-ops. Completion never replays a skipped phase or waits for earlier actions. While latched: @@ -536,12 +569,13 @@ The topology drain is therefore only an FSM rearm signal. A successful, structurally valid result for the same deployment with an earlier status follows the existing RDS BGD backward-transition behavior. A successful empty result or confirmed table absence before completion is treated -as cancellation. Either condition runs idempotent rollback: +as cancellation. Either condition runs effect-driven cleanup in rollback mode: 1. Remove every traffic pin that was applied. 2. Drain/purge affected production-hostname pools so subsequent connections use restored canonical DNS. -3. Restore the production writer to the writer hostgroup when it was demoted. +3. Reconcile the mapped production writer into the writer hostgroup. This is a + no-op if the writer is already there or the map is empty. 4. Restore its reader placement according to `writer_is_also_reader`. 5. Resume normal production Aurora monitoring. 6. Return topology probing to production members and the appropriate cadence. @@ -569,16 +603,16 @@ A worker that first observes INITIATED, IN_PROGRESS, or POST_PROCESSING rebuilds all prerequisites before applying that phase's actions. In particular, POST_PROCESSING cannot pin traffic until it has a complete target snapshot. -A worker that first observes TARGET-only completion performs completion cleanup -once, even when no traffic pins were recorded, retains the fingerprint, and -enters the terminal latch. It must not manufacture or replay earlier phase -actions. +A worker that first observes TARGET-only completion has an empty member map. +Its member-scoped completion actions are consequently no-ops. The worker +retains the fingerprint and enters the terminal latch without manufacturing or +replaying earlier phase actions. ### 11.2 Configuration and variable refresh An unrelated `LOAD MYSQL SERVERS TO RUNTIME` or variable refresh must preserve the FSM status, deployment fingerprint, complete snapshots, cached IPs, applied -pin flags, writer-demotion state, probe-suspension state, and terminal latch. +pin flags, probe-suspension state, and terminal latch. Changing configured green hostgroups refreshes the staging/pool references but does not restart an active deployment from `NONE`. Disabling the global @@ -671,7 +705,16 @@ The existing RDS Multi-AZ builder and FSM remain unchanged. - No target-writability query gates POST_PROCESSING. - Readers remain eligible and are never shunned. - Repeated POST_PROCESSING retries only unapplied work. -- First TARGET completion removes all pins and cleans up immediately. +- The per-pair action flag records application of pinning and retirement + marking, not asynchronous physical connection closure. +- First TARGET completion runs immediate effect-driven cleanup using the + existing map and worker state. +- Direct completion with an empty map performs no production-member routing or + retirement action and enters the completed latch. +- Completion after IN_PROGRESS restores the writer even when POST_PROCESSING + was not observed. +- Completion after POST_PROCESSING removes applied pins without reapplying + connection retirement or waiting for marked connections to close. - Completion performs no DNS verification and does not wait for table drain. - Repeated completion is a no-op while latched. - Empty/absent topology releases the latch and returns to `NONE`. From e06fa1afb60a4026cde6396c773a26b202c572b9 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 09:45:03 +0000 Subject: [PATCH 09/26] feat: add Aurora BGD hostgroup schema --- include/MySQL_HostGroups_Manager.h | 18 +- include/ProxySQL_Admin_Tables_Definitions.h | 44 ++- lib/MySQL_HostGroups_Manager.cpp | 269 ++++++++++++++++-- lib/ProxySQL_Admin.cpp | 39 ++- lib/ProxySQL_Admin_Disk_Upgrade.cpp | 22 ++ test/tap/tests/unit/Makefile | 1 + .../tests/unit/admin_disk_upgrade_unit-t.cpp | 34 ++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 242 ++++++++++++++++ 8 files changed, 611 insertions(+), 58 deletions(-) create mode 100644 test/tap/tests/unit/aurora_bgd_config_unit-t.cpp diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 6edc571723..fabdb6b5a6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "ev.h" #include "wqueue.h" @@ -52,6 +54,8 @@ #define MYHGM_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" #define MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL DEFAULT '' , " \ "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ @@ -62,7 +66,7 @@ "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , " \ "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , " \ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ - "comment VARCHAR , UNIQUE (reader_hostgroup))" + "comment VARCHAR , bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" #define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ @@ -379,6 +383,18 @@ class AWS_Aurora_Info { AWS_Aurora_Info& operator=(const AWS_Aurora_Info&) = delete; }; +/** + * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. + * + * The returned result contains only valid rows, ordered like the Aurora Admin + * table. Legacy projections without green hostgroups are normalized with both + * green values set to SQL NULL. The caller owns the returned result. + */ +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +); + struct p_hg_counter { enum metric { servers_table_version = 0, diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 4514853bd0..164ec01856 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -235,9 +235,47 @@ // V2_0_10: Adds autopurge_missing_checks column #define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" +// V3_0_12: Adds nullable Aurora blue/green staging hostgroups. +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 "CREATE TABLE mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , "\ + "bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" // AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. // The runtime table carries one extra runtime-only column: auto_generated. diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 728550102a..be580d71c6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -30,6 +30,7 @@ using json = nlohmann::json; #include #include +#include #include using std::function; @@ -58,6 +59,198 @@ class MyHGC; const int MYSQL_ERRORS_STATS_FIELD_NUM = 11; +namespace { + +constexpr int AWS_AURORA_CONFIG_COLUMN_COUNT = 17; + +const char* const AWS_AURORA_CONFIG_COLUMNS[AWS_AURORA_CONFIG_COLUMN_COUNT] = { + "writer_hostgroup", + "reader_hostgroup", + "green_writer_hostgroup", + "green_reader_hostgroup", + "active", + "aurora_port", + "domain_name", + "max_lag_ms", + "check_interval_ms", + "check_timeout_ms", + "writer_is_also_reader", + "new_reader_weight", + "add_lag_ms", + "min_lag_ms", + "lag_num_checks", + "autopurge_missing_checks", + "comment" +}; + +struct Aurora_Hostgroup_Role { + const char* field; + int value; +}; + +struct Aurora_Config_Row { + std::vector fields; + std::vector roles; + std::vector errors; + int writer_hostgroup; + bool active; + bool locally_valid; +}; + +int aws_aurora_column_index(const SQLite3_result* candidate, const char* name) { + for (size_t i = 0; i < candidate->column_definition.size(); ++i) { + if (strcasecmp(candidate->column_definition[i]->name, name) == 0) { + return static_cast(i); + } + } + return -1; +} + +std::string aws_aurora_nullable_value(const char* value) { + return value ? value : "NULL"; +} + +} // namespace + +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +) { + SQLite3_result* filtered = new SQLite3_result(AWS_AURORA_CONFIG_COLUMN_COUNT); + for (const char* column : AWS_AURORA_CONFIG_COLUMNS) { + filtered->add_column_definition(SQLITE_TEXT, column); + } + + if (candidate == nullptr) { + return filtered; + } + + std::vector source_indexes(AWS_AURORA_CONFIG_COLUMN_COUNT, -1); + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + source_indexes[i] = aws_aurora_column_index(candidate, AWS_AURORA_CONFIG_COLUMNS[i]); + } + + // Older peers send the pre-BGD 15-column projection. Treat the two absent + // green fields as SQL NULL until cluster synchronization is upgraded. + const bool green_writer_absent = source_indexes[2] == -1; + const bool green_reader_absent = source_indexes[3] == -1; + if (green_writer_absent != green_reader_absent) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection contains only one green hostgroup column" + ); + return filtered; + } + + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + if ((i == 2 || i == 3) && green_writer_absent) { + continue; + } + if (source_indexes[i] == -1) { + errors.emplace_back( + std::string("mysql_aws_aurora_hostgroups rejected: candidate projection is missing ") + + AWS_AURORA_CONFIG_COLUMNS[i] + ); + return filtered; + } + } + + std::vector rows; + rows.reserve(candidate->rows.size()); + + for (const SQLite3_row* source_row : candidate->rows) { + Aurora_Config_Row row; + row.fields.resize(AWS_AURORA_CONFIG_COLUMN_COUNT, nullptr); + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + if (source_indexes[i] != -1) { + row.fields[i] = source_row->fields[source_indexes[i]]; + } + } + + row.writer_hostgroup = row.fields[0] ? atoi(row.fields[0]) : -1; + row.active = row.fields[4] && atoi(row.fields[4]) != 0; + row.locally_valid = true; + + const bool has_green_writer = row.fields[2] != nullptr; + const bool has_green_reader = row.fields[3] != nullptr; + if (has_green_writer != has_green_reader) { + std::ostringstream message; + message << "green_writer_hostgroup=" << aws_aurora_nullable_value(row.fields[2]) + << " and green_reader_hostgroup=" << aws_aurora_nullable_value(row.fields[3]) + << " must both be NULL or both be non-NULL"; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + + row.roles.push_back({"writer_hostgroup", row.writer_hostgroup}); + row.roles.push_back({"reader_hostgroup", row.fields[1] ? atoi(row.fields[1]) : -1}); + if (has_green_writer && has_green_reader) { + row.roles.push_back({"green_writer_hostgroup", atoi(row.fields[2])}); + row.roles.push_back({"green_reader_hostgroup", atoi(row.fields[3])}); + } + + if (row.locally_valid) { + for (size_t left = 0; left < row.roles.size(); ++left) { + for (size_t right = left + 1; right < row.roles.size(); ++right) { + if (row.roles[left].value == row.roles[right].value) { + std::ostringstream message; + message << "conflicting fields " << row.roles[left].field << '=' << row.roles[left].value + << " and " << row.roles[right].field << '=' << row.roles[right].value; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + } + } + } + + rows.emplace_back(std::move(row)); + } + + // A locally invalid row is already absent from the replacement and must not + // make an otherwise valid row fail. Cross-row conflicts are directional: + // every row is checked against the roles owned by every other active row. + for (size_t row_index = 0; row_index < rows.size(); ++row_index) { + Aurora_Config_Row& row = rows[row_index]; + if (!row.locally_valid) { + continue; + } + + for (size_t other_index = 0; other_index < rows.size(); ++other_index) { + const Aurora_Config_Row& other = rows[other_index]; + if (row_index == other_index || !other.locally_valid || !other.active) { + continue; + } + + for (const Aurora_Hostgroup_Role& role : row.roles) { + for (const Aurora_Hostgroup_Role& other_role : other.roles) { + if (role.value == other_role.value) { + std::ostringstream message; + message << "conflicting field " << role.field << '=' << role.value + << " overlaps " << other_role.field << '=' << other_role.value + << " in writer_hostgroup=" << other.writer_hostgroup; + row.errors.emplace_back(message.str()); + } + } + } + } + } + + for (Aurora_Config_Row& row : rows) { + if (row.errors.empty()) { + filtered->add_row(row.fields.data()); + continue; + } + + for (const std::string& reason : row.errors) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups writer_hostgroup=" << row.writer_hostgroup + << " rejected: " << reason; + errors.emplace_back(message.str()); + } + } + + return filtered; +} + struct ev_io * new_connect_watcher(char *address, uint16_t gtid_port, uint16_t mysql_port); void * GTID_syncer_run(); @@ -6422,9 +6615,9 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { } int rc; //sqlite3 *mydb3=mydb->get_db(); - char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," + char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," "check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; auto [rc1, statement_unique] = mydb->prepare_v2(query); ASSERT_SQLITE_OK(rc1, mydb); sqlite3_stmt *statement = statement_unique.get(); @@ -6439,34 +6632,50 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); - int active=atoi(r->fields[2]); - int aurora_port = atoi(r->fields[3]); - int max_lag_ms = atoi(r->fields[5]); - int check_interval_ms = atoi(r->fields[6]); - int check_timeout_ms = atoi(r->fields[7]); - int writer_is_also_reader = atoi(r->fields[8]); - int new_reader_weight = atoi(r->fields[9]); - int add_lag_ms = atoi(r->fields[10]); - int min_lag_ms = atoi(r->fields[11]); - int lag_num_checks = atoi(r->fields[12]); - int autopurge_missing_checks = atoi(r->fields[13]); - proxy_info("Loading AWS Aurora info for (%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup,(active ? "on" : "off"),aurora_port, - r->fields[4],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks,check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[14]); + const char* green_writer_hostgroup = r->fields[2]; + const char* green_reader_hostgroup = r->fields[3]; + int active=atoi(r->fields[4]); + int aurora_port = atoi(r->fields[5]); + int max_lag_ms = atoi(r->fields[7]); + int check_interval_ms = atoi(r->fields[8]); + int check_timeout_ms = atoi(r->fields[9]); + int writer_is_also_reader = atoi(r->fields[10]); + int new_reader_weight = atoi(r->fields[11]); + int add_lag_ms = atoi(r->fields[12]); + int min_lag_ms = atoi(r->fields[13]); + int lag_num_checks = atoi(r->fields[14]); + int autopurge_missing_checks = atoi(r->fields[15]); + proxy_info("Loading AWS Aurora info for (%d,%d,%s,%s,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup ? green_writer_hostgroup : "NULL", green_reader_hostgroup ? green_reader_hostgroup : "NULL", + (active ? "on" : "off"),aurora_port,r->fields[6],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks, + check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[16]); rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, aurora_port); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(green_writer_hostgroup)); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, mydb); + if (green_reader_hostgroup) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(green_reader_hostgroup)); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, aurora_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -6477,12 +6686,12 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (it2!=AWS_Aurora_Info_Map.end()) { info=it2->second; bool changed=false; - changed=info->update(reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + changed=info->update(reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); if (changed) { //info->need_converge=true; } } else { - info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); //info->need_converge=true; AWS_Aurora_Info_Map.insert(AWS_Aurora_Info_Map.begin(), std::pair(writer_hostgroup,info)); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index b7a9ce5562..edf310dc6f 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -8110,27 +8110,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // support for AWS Aurora, table mysql_aws_aurora_hostgroups - - // look for invalid combinations - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a JOIN mysql_aws_aurora_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", query, error); - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - proxy_error("Incompatible entry in mysql_aws_aurora_hostgroups will be ignored : ( %s , %s , %s , %s )\n", r->fields[0], r->fields[1], r->fields[2], r->fields[3]); - } - } - if (resultset) delete resultset; - resultset=NULL; - -//#ifdef TEST_AURORA // temporary enabled only for testing purpose - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; -//#else -// query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a WHERE 1=0"; -//#endif + query=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); if (incoming_aurora_hostgroups == nullptr) { admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora); @@ -8140,8 +8124,17 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc if (error) { proxy_error("Error on %s : %s\n", query, error); } else { - // Pass the resultset to MyHGM - MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); + std::vector validation_errors; + SQLite3_result* filtered_aws_aurora = validate_and_filter_aws_aurora_hostgroups( + resultset_aws_aurora, + validation_errors + ); + for (const std::string& validation_error : validation_errors) { + proxy_error("%s\n", validation_error.c_str()); + } + delete resultset_aws_aurora; + resultset_aws_aurora = nullptr; + MyHGM->save_incoming_mysql_table(filtered_aws_aurora,"mysql_aws_aurora_hostgroups"); } // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups diff --git a/lib/ProxySQL_Admin_Disk_Upgrade.cpp b/lib/ProxySQL_Admin_Disk_Upgrade.cpp index fbf3f2b638..b8cb964210 100644 --- a/lib/ProxySQL_Admin_Disk_Upgrade.cpp +++ b/lib/ProxySQL_Admin_Disk_Upgrade.cpp @@ -504,6 +504,28 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { "add_lag_ms, min_lag_ms, lag_num_checks, comment FROM mysql_aws_aurora_hostgroups_v209"); } + // Upgrade mysql_aws_aurora_hostgroups from V2_0_10 to V3_0_12 + // (adds nullable green_writer_hostgroup and green_reader_hostgroup columns). + rci=configdb->check_table_structure("mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + if (rci) { + proxy_warning("Detected version pre-v3.0.12 of mysql_aws_aurora_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_aws_aurora_hostgroups in progress\n"); + configdb->execute("DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_v2010"); + configdb->execute("ALTER TABLE mysql_aws_aurora_hostgroups RENAME TO mysql_aws_aurora_hostgroups_v2010"); + configdb->build_table("mysql_aws_aurora_hostgroups",ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS,false); + configdb->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment" + ") SELECT " + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment " + "FROM mysql_aws_aurora_hostgroups_v2010" + ); + } + // upgrade mysql_hostgroup_attributes rci=configdb->check_table_structure("mysql_hostgroup_attributes",ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0); if (rci) { diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index 737a93b037..3bc9d2fce2 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -428,6 +428,7 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ gtid_set_unit-t \ gtid_server_data_unit-t \ admin_disk_upgrade_unit-t \ + aurora_bgd_config_unit-t \ glovars_unit-t \ pgsql_servers_ssl_params_unit-t \ connection_unhealthy_unit-t \ diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index f900625e6f..d223105327 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -382,6 +382,37 @@ static void test_mysql_servers_no_upgrade_needed() { ok(count == 1, "mysql_servers: data unchanged when no upgrade needed (got %d)", count); } +static void test_aurora_hostgroups_upgrade_from_v2_0_10() { + TestDiskUpgrade t; + SQLite3DB *db = t.db(); + + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight," + "add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (10,20,1,3307,'.cluster.example',125,1500,900,1,7,40,20,3,9,'existing Aurora row')" + ); + + t.upgrade_mysql_servers(); + + ok(table_matches_current(db, "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS), + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade produces current schema"); + ok(query_int(db, "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade preserves the row"); + ok(query_int(db, "SELECT green_writer_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green writer is NULL"); + ok(query_int(db, "SELECT green_reader_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green reader is NULL"); + ok(query_string(db, "SELECT domain_name FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == ".cluster.example", + "mysql_aws_aurora_hostgroups: migration preserves configured values"); + ok(query_int(db, "SELECT autopurge_missing_checks FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 9, + "mysql_aws_aurora_hostgroups: migration preserves autopurge_missing_checks"); + ok(query_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='mysql_aws_aurora_hostgroups_v2010'") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 table is retained as v2010"); +} + // ============================================================================ // disk_upgrade_pgsql_replication_hostgroups() tests // ============================================================================ @@ -549,7 +580,7 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { // ============================================================================ int main() { - plan(60); + plan(67); test_init_minimal(); // scheduler tests @@ -571,6 +602,7 @@ int main() { // mysql_servers tests test_mysql_servers_upgrade_from_v1_1_0(); test_mysql_servers_no_upgrade_needed(); + test_aurora_hostgroups_upgrade_from_v2_0_10(); // pgsql_replication_hostgroups tests test_pgsql_repl_hg_upgrade_from_v3_0_1(); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp new file mode 100644 index 0000000000..ad078e4cd0 --- /dev/null +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -0,0 +1,242 @@ +/** + * @file aurora_bgd_config_unit-t.cpp + * @brief Aurora blue/green hostgroup schema and LOAD validation contracts. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "cpp.h" +#include "MySQL_HostGroups_Manager.h" +#include "ProxySQL_Admin_Tables_Definitions.h" + +#include +#include +#include + +static std::string query_string(SQLite3DB* db, const char* query) { + char* error = nullptr; + SQLite3_result* result = db->execute_statement(query, &error); + std::string value; + if (result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int query_int(SQLite3DB* db, const char* query) { + return db->return_one_int(query); +} + +static SQLite3DB* make_database() { + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + return db; +} + +static const char* const AURORA_COLUMNS[] = { + "writer_hostgroup", "reader_hostgroup", "green_writer_hostgroup", "green_reader_hostgroup", + "active", "aurora_port", "domain_name", "max_lag_ms", "check_interval_ms", + "check_timeout_ms", "writer_is_also_reader", "new_reader_weight", "add_lag_ms", + "min_lag_ms", "lag_num_checks", "autopurge_missing_checks", "comment" +}; + +static SQLite3_result* make_candidate(bool legacy = false) { + SQLite3_result* result = new SQLite3_result(legacy ? 15 : 17); + for (size_t i = 0; i < 17; ++i) { + if (legacy && (i == 2 || i == 3)) { + continue; + } + result->add_column_definition(SQLITE_TEXT, AURORA_COLUMNS[i]); + } + return result; +} + +static void add_candidate_row( + SQLite3_result* result, + int writer, + int reader, + const char* green_writer, + const char* green_reader, + bool active, + const char* comment = "test" +) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), green_writer, green_reader, + active ? "1" : "0", "3306", ".cluster.example", "600000", "1000", "800", + "0", "1", "30", "30", "1", "0", comment + }; + result->add_row(fields); +} + +static void add_legacy_candidate_row(SQLite3_result* result, int writer, int reader) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), "1", "3306", ".legacy.example", "600000", + "1000", "800", "0", "1", "30", "30", "1", "0", "legacy" + }; + result->add_row(fields); +} + +static bool contains_error(const std::vector& errors, const std::string& first, const std::string& second = "") { + for (const std::string& error : errors) { + if (error.find(first) != std::string::npos && + (second.empty() || error.find(second) != std::string::npos)) { + return true; + } + } + return false; +} + +static bool contains_writer(SQLite3_result* result, int writer) { + for (SQLite3_row* row : result->rows) { + if (row->fields[0] && atoi(row->fields[0]) == writer) { + return true; + } + } + return false; +} + +static void test_schema_contract() { + SQLite3DB* db = make_database(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + + const std::string configured_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + const std::string runtime_columns = configured_columns + ",bgd_status"; + + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == configured_columns, + "Aurora configured columns follow the BGD contract order"); + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups')") == runtime_columns, + "Aurora runtime columns append bgd_status"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "configured green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "configured green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "runtime green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "runtime green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "runtime bgd_status defaults to NONE"); + ok(query_int(db, "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "bgd_status is absent from configured Aurora rows"); + ok(!db->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,-1,3,'.negative.example')"), + "configured schema rejects a negative green writer hostgroup"); + ok(!db->execute("INSERT INTO runtime_mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,3,-1,'.negative.example')"), + "runtime schema rejects a negative green reader hostgroup"); + delete db; + + SQLite3DB* hgm_db = make_database(); + hgm_db->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(query_string(hgm_db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == runtime_columns, + "HGM Aurora columns match the runtime contract"); + ok(query_string(hgm_db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "HGM bgd_status defaults to NONE"); + delete hgm_db; +} + +static void test_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 10, 20, nullptr, nullptr, true); + add_candidate_row(candidate, 30, 40, "31", "41", true); + add_candidate_row(candidate, 50, 60, "51", nullptr, true); + add_candidate_row(candidate, 70, 80, "70", "81", true); + add_candidate_row(candidate, 90, 100, "91", "101", true); + add_candidate_row(candidate, 110, 120, "111", "90", true); + add_candidate_row(candidate, 130, 140, nullptr, nullptr, true); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 3, "mixed LOAD publishes only its three valid Aurora rows"); + ok(contains_writer(filtered, 10), "paired NULL green hostgroups are valid"); + ok(contains_writer(filtered, 30), "four distinct configured hostgroups are valid"); + ok(contains_writer(filtered, 130), "an unrelated valid row survives other validation failures"); + ok(contains_error(errors, "writer_hostgroup=50", "green_reader_hostgroup=NULL"), + "mixed NULL validation identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=70", "conflicting fields"), + "same-row overlap identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=90", "writer_hostgroup=110"), + "first active row in a cross-row conflict identifies the other writer"); + ok(contains_error(errors, "writer_hostgroup=110", "writer_hostgroup=90"), + "second active row in a cross-row conflict identifies the other writer"); + delete filtered; + delete candidate; +} + +static void test_inactive_cross_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 150, 160, nullptr, nullptr, true); + add_candidate_row(candidate, 170, 180, "171", "150", false); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 1, "an inactive row conflicting with an active row is isolated"); + ok(contains_writer(filtered, 150), "the active owner survives an inactive conflicting row"); + ok(contains_error(errors, "writer_hostgroup=170", "writer_hostgroup=150"), + "inactive conflict reports the active owner"); + ok(!contains_error(errors, "mysql_aws_aurora_hostgroups writer_hostgroup=150 rejected:"), + "inactive roles do not invalidate the active owner"); + delete filtered; + delete candidate; +} + +static void test_legacy_projection_normalization() { + SQLite3_result* candidate = make_candidate(true); + add_legacy_candidate_row(candidate, 190, 200); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->columns == 17 && filtered->rows_count == 1, + "legacy Aurora projection is normalized to the configured BGD shape"); + ok(filtered->rows[0]->fields[2] == nullptr && filtered->rows[0]->fields[3] == nullptr, + "legacy projection normalizes both green hostgroups to SQL NULL"); + ok(errors.empty(), "legacy Aurora projection remains valid"); + delete filtered; + delete candidate; +} + +static void test_invalid_replacement_removes_previous_row() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 210, 220, "211", "221", true); + std::vector initial_errors; + SQLite3_result* initial_filtered = validate_and_filter_aws_aurora_hostgroups(initial, initial_errors); + ok(initial_filtered->rows_count == 1, "initial valid Aurora row is publishable"); + + SQLite3_result* replacement = make_candidate(); + add_candidate_row(replacement, 210, 220, "211", nullptr, true); + std::vector replacement_errors; + SQLite3_result* replacement_filtered = validate_and_filter_aws_aurora_hostgroups(replacement, replacement_errors); + ok(replacement_filtered->rows_count == 0, + "invalid replacement omits a previously published writer from the atomic replacement"); + ok(contains_error(replacement_errors, "writer_hostgroup=210", "must both be NULL"), + "invalid replacement reports its former writer owner"); + + delete replacement_filtered; + delete replacement; + delete initial_filtered; + delete initial; +} + +int main() { + plan(30); + test_init_minimal(); + + test_schema_contract(); // 12 + test_row_validation(); // 8 + test_inactive_cross_row_validation(); // 4 + test_legacy_projection_normalization(); // 3 + test_invalid_replacement_removes_previous_row(); // 3 + + test_cleanup_minimal(); + return exit_status(); +} From 5a68861bfc8bc60820fbe1ed70bd88f9f6ba4e6b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:05:47 +0000 Subject: [PATCH 10/26] feat: publish Aurora BGD runtime status --- include/MySQL_HostGroups_Manager.h | 14 +- lib/MySQL_HostGroups_Manager.cpp | 218 ++++++++++++++---- lib/ProxySQL_Admin.cpp | 29 ++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 140 ++++++++++- 4 files changed, 347 insertions(+), 54 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index fabdb6b5a6..4d1c56aafd 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -360,6 +360,8 @@ class AWS_Aurora_Info { public: int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int aurora_port; int max_lag_ms; int add_lag_ms; @@ -376,8 +378,8 @@ class AWS_Aurora_Info { char * comment; bool active; bool active_; - AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); - bool update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + bool update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); ~AWS_Aurora_Info(); AWS_Aurora_Info(const AWS_Aurora_Info&) = delete; AWS_Aurora_Info& operator=(const AWS_Aurora_Info&) = delete; @@ -786,6 +788,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { public: // Friend declaration for WebUI monitoring metrics collector friend class ProxySQL::Monitoring::MetricsCollector; + friend class TestAuroraBGDRuntime; std::mutex galera_set_writer_mutex; /** @@ -1120,6 +1123,13 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param status AWS_RDS_BGD_Status underlying value. */ void aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status); + /** + * @brief Publish the node-local Aurora BGD state for one runtime row. + * + * Invalid status strings and writer hostgroups not present at runtime are + * ignored. Configuration reloads do not write this column. + */ + void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index be580d71c6..1a1c0b01c5 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -9,6 +9,7 @@ using json = nlohmann::json; #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" +#include #include #include #include @@ -1737,8 +1738,6 @@ bool MySQL_HostGroups_Manager::commit( // AWS Aurora if (incoming_aws_aurora_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); generate_mysql_aws_aurora_hostgroups_table(); } @@ -2452,8 +2451,8 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { char * query = (char *)""; if (name == "mysql_aws_aurora_hostgroups") { - query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," - "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," "check_interval_ms,check_timeout_ms,comment,auto_generated,status FROM mysql_aws_rds_bgd_hostgroups"; @@ -4181,6 +4180,38 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write wrunlock(); } +void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { + static const std::array valid_statuses { + "NONE", + "AVAILABLE", + "SWITCHOVER_INITIATED", + "SWITCHOVER_IN_PROGRESS", + "SWITCHOVER_IN_POST_PROCESSING", + "SWITCHOVER_COMPLETED" + }; + + bool valid = std::any_of(valid_statuses.begin(), valid_statuses.end(), [&bgd_status](const char* status) { + return bgd_status == status; + }); + if (!valid) { + proxy_error("Invalid AWS Aurora BGD runtime status '%s' for writer hostgroup %d\n", + bgd_status.c_str(), writer_hostgroup); + return; + } + + wrlock(); + const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; + auto [prepare_rc, statement_unique] = mydb->prepare_v2(query); + ASSERT_SQLITE_OK(prepare_rc, mydb); + sqlite3_stmt* statement = statement_unique.get(); + int rc = (*proxy_sqlite3_bind_text)(statement, 1, bgd_status.c_str(), -1, SQLITE_TRANSIENT); + ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, writer_hostgroup); + ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + wrunlock(); +} + /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -6223,13 +6254,15 @@ SQLite3_result * MySQL_HostGroups_Manager::get_mysql_errors(bool reset) { return result; } -AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { comment=NULL; if (c) { comment=strdup(c); } writer_hostgroup=w; reader_hostgroup=r; + green_writer_hostgroup=gw; + green_reader_hostgroup=gr; max_lag_ms=maxl; add_lag_ms=al; min_lag_ms=minl; @@ -6257,13 +6290,21 @@ AWS_Aurora_Info::~AWS_Aurora_Info() { } } -bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +bool AWS_Aurora_Info::update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { bool ret=false; active_=true; if (reader_hostgroup!=r) { reader_hostgroup=r; ret=true; } + if (green_writer_hostgroup!=gw) { + green_writer_hostgroup=gw; + ret=true; + } + if (green_reader_hostgroup!=gr) { + green_reader_hostgroup=gr; + ret=true; + } if (max_lag_ms!=maxl) { max_lag_ms=maxl; ret=true; @@ -6613,14 +6654,81 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (incoming_aws_aurora_hostgroups==NULL) { return; } + + struct RuntimeRow { + int reader_hostgroup; + std::string bgd_status; + }; + + std::map runtime_rows; + std::map incoming_reader_hostgroups; + for (SQLite3_row* row : incoming_aws_aurora_hostgroups->rows) { + incoming_reader_hostgroups.emplace(atoi(row->fields[0]), atoi(row->fields[1])); + } + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* select_query = + "SELECT writer_hostgroup, reader_hostgroup, bgd_status FROM mysql_aws_aurora_hostgroups"; + mydb->execute_statement(select_query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", select_query, error); + free(error); + assert(0); + } + if (resultset) { + for (SQLite3_row* row : resultset->rows) { + runtime_rows.emplace( + atoi(row->fields[0]), + RuntimeRow {atoi(row->fields[1]), row->fields[2] ? row->fields[2] : "NONE"} + ); + } + delete resultset; + } + int rc; - //sqlite3 *mydb3=mydb->get_db(); - char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," - "check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; - auto [rc1, statement_unique] = mydb->prepare_v2(query); - ASSERT_SQLITE_OK(rc1, mydb); - sqlite3_stmt *statement = statement_unique.get(); + const char* delete_query = "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=?1"; + auto [delete_rc, delete_statement_unique] = mydb->prepare_v2(delete_query); + ASSERT_SQLITE_OK(delete_rc, mydb); + sqlite3_stmt* delete_statement = delete_statement_unique.get(); + + // Delete missing deployments and release a changed reader hostgroup before inserting its replacement. + for (const auto& [writer_hostgroup, runtime_row] : runtime_rows) { + auto incoming_it = incoming_reader_hostgroups.find(writer_hostgroup); + bool removed = incoming_it == incoming_reader_hostgroups.end(); + bool reader_changed = !removed && incoming_it->second != runtime_row.reader_hostgroup; + if (!removed && !reader_changed) { + continue; + } + rc=(*proxy_sqlite3_bind_int64)(delete_statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(delete_statement); + rc=(*proxy_sqlite3_clear_bindings)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + } + + const char* update_query = + "UPDATE mysql_aws_aurora_hostgroups SET " + "reader_hostgroup=?1, green_writer_hostgroup=?2, green_reader_hostgroup=?3, active=?4, " + "aurora_port=?5, domain_name=?6, max_lag_ms=?7, check_interval_ms=?8, check_timeout_ms=?9, " + "writer_is_also_reader=?10, new_reader_weight=?11, add_lag_ms=?12, min_lag_ms=?13, " + "lag_num_checks=?14, autopurge_missing_checks=?15, comment=?16 " + "WHERE writer_hostgroup=?17"; + auto [update_rc, update_statement_unique] = mydb->prepare_v2(update_query); + ASSERT_SQLITE_OK(update_rc, mydb); + sqlite3_stmt* update_statement = update_statement_unique.get(); + + const char* insert_query = + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, " + "aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, " + "new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment, bgd_status" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; + auto [insert_rc, insert_statement_unique] = mydb->prepare_v2(insert_query); + ASSERT_SQLITE_OK(insert_rc, mydb); + sqlite3_stmt* insert_statement = insert_statement_unique.get(); + proxy_info("New mysql_aws_aurora_hostgroups table\n"); pthread_mutex_lock(&AWS_Aurora_Info_mutex); for (std::map::iterator it1 = AWS_Aurora_Info_Map.begin() ; it1 != AWS_Aurora_Info_Map.end(); ++it1) { @@ -6632,8 +6740,10 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); - const char* green_writer_hostgroup = r->fields[2]; - const char* green_reader_hostgroup = r->fields[3]; + const char* green_writer_value = r->fields[2]; + const char* green_reader_value = r->fields[3]; + int green_writer_hostgroup = green_writer_value ? atoi(green_writer_value) : -1; + int green_reader_hostgroup = green_reader_value ? atoi(green_reader_value) : -1; int active=atoi(r->fields[4]); int aurora_port = atoi(r->fields[5]); int max_lag_ms = atoi(r->fields[7]); @@ -6645,37 +6755,54 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { int min_lag_ms = atoi(r->fields[13]); int lag_num_checks = atoi(r->fields[14]); int autopurge_missing_checks = atoi(r->fields[15]); - proxy_info("Loading AWS Aurora info for (%d,%d,%s,%s,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, - green_writer_hostgroup ? green_writer_hostgroup : "NULL", green_reader_hostgroup ? green_reader_hostgroup : "NULL", + proxy_info("Loading AWS Aurora info for (%d,%d,%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup, green_reader_hostgroup, (active ? "on" : "off"),aurora_port,r->fields[6],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks, check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[16]); - rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - if (green_writer_hostgroup) { - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(green_writer_hostgroup)); + + auto runtime_it = runtime_rows.find(writer_hostgroup); + bool update_existing = + runtime_it != runtime_rows.end() && + runtime_it->second.reader_hostgroup == reader_hostgroup; + sqlite3_stmt* statement = update_existing ? update_statement : insert_statement; + int field_offset = update_existing ? 0 : 1; + if (!update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1 + field_offset, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 2 + field_offset, green_writer_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 3); + rc=(*proxy_sqlite3_bind_null)(statement, 2 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); - if (green_reader_hostgroup) { - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(green_reader_hostgroup)); + if (green_reader_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3 + field_offset, green_reader_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 4); + rc=(*proxy_sqlite3_bind_null)(statement, 3 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, aurora_port); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 15, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 16, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4 + field_offset, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5 + field_offset, aurora_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 6 + field_offset, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7 + field_offset, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8 + field_offset, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9 + field_offset, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10 + field_offset, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11 + field_offset, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12 + field_offset, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13 + field_offset, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14 + field_offset, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15 + field_offset, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 16 + field_offset, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 17, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } else { + const std::string bgd_status = runtime_it == runtime_rows.end() + ? std::string("NONE") + : runtime_it->second.bgd_status; + rc=(*proxy_sqlite3_bind_text)(statement, 18, bgd_status.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -6686,12 +6813,12 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (it2!=AWS_Aurora_Info_Map.end()) { info=it2->second; bool changed=false; - changed=info->update(reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); + changed=info->update(reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); if (changed) { //info->need_converge=true; } } else { - info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); + info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); //info->need_converge=true; AWS_Aurora_Info_Map.insert(AWS_Aurora_Info_Map.begin(), std::pair(writer_hostgroup,info)); } @@ -6713,9 +6840,11 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { // it is now time to build a new structure in Monitor - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - update_aws_aurora_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); + if (GloMyMon) { + pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); + update_aws_aurora_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); + } pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } @@ -7414,7 +7543,8 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," - " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name FROM mysql_servers" + " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name," + " green_writer_hostgroup, green_reader_hostgroup FROM mysql_servers" " JOIN mysql_aws_aurora_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index edf310dc6f..087b786d28 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7539,9 +7539,9 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { char *query=NULL; if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; } else { - query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; } auto [rc1, statement_unique] = admindb->prepare_v2(query); @@ -7553,11 +7553,21 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { SQLite3_row *r=*it; rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, atoi(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); @@ -7565,7 +7575,12 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 13, atoi(r->fields[12])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 14, atoi(r->fields[13])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, atoi(r->fields[14])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, atoi(r->fields[15])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (_runtime) { + rc=(*proxy_sqlite3_bind_text)(statement, 18, r->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index ad078e4cd0..e4adee5c4f 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -13,8 +13,39 @@ #include #include +#include #include +extern MySQL_HostGroups_Manager* MyHGM; + +class TestAuroraBGDRuntime { +public: + static void reload(MySQL_HostGroups_Manager* hgm, SQLite3_result* candidate) { + hgm->wrlock(); + hgm->save_incoming_mysql_table(candidate, "mysql_aws_aurora_hostgroups"); + hgm->generate_mysql_aws_aurora_hostgroups_table(); + hgm->wrunlock(); + } + + static bool green_hostgroups( + MySQL_HostGroups_Manager* hgm, + int writer_hostgroup, + int& green_writer_hostgroup, + int& green_reader_hostgroup + ) { + pthread_mutex_lock(&hgm->AWS_Aurora_Info_mutex); + auto info_it = hgm->AWS_Aurora_Info_Map.find(writer_hostgroup); + if (info_it == hgm->AWS_Aurora_Info_Map.end()) { + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return false; + } + green_writer_hostgroup = info_it->second->green_writer_hostgroup; + green_reader_hostgroup = info_it->second->green_reader_hostgroup; + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return true; + } +}; + static std::string query_string(SQLite3DB* db, const char* query) { char* error = nullptr; SQLite3_result* result = db->execute_statement(query, &error); @@ -31,6 +62,23 @@ static int query_int(SQLite3DB* db, const char* query) { return db->return_one_int(query); } +static std::string hgm_query_string(const char* query) { + char* error = nullptr; + SQLite3_result* result = MyHGM->execute_query(const_cast(query), &error); + std::string value; + if (!error && result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int hgm_query_int(const char* query) { + const std::string value = hgm_query_string(query); + return value.empty() ? 0 : atoi(value.c_str()); +} + static SQLite3DB* make_database() { SQLite3DB* db = new SQLite3DB(); db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); @@ -227,8 +275,94 @@ static void test_invalid_replacement_removes_previous_row() { delete initial; } +static void test_runtime_ownership_and_status() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 300, 310, "301", "311", true); + add_candidate_row(initial, 320, 330, nullptr, nullptr, true); + TestAuroraBGDRuntime::reload(MyHGM, initial); + + int green_writer = 0; + int green_reader = 0; + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 301 && green_reader == 311, + "Aurora runtime info owns configured green hostgroups"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 320, green_writer, green_reader) && + green_writer == -1 && green_reader == -1, + "SQL NULL green hostgroups use the -1 runtime sentinel"); + ok(hgm_query_string( + "SELECT GROUP_CONCAT(bgd_status, ',') FROM " + "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" + ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); + + SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); + ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); + delete dump; + + MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); + const char* accepted_statuses[] = { + "NONE", + "AVAILABLE", + "SWITCHOVER_INITIATED", + "SWITCHOVER_IN_PROGRESS", + "SWITCHOVER_IN_POST_PROCESSING", + "SWITCHOVER_COMPLETED" + }; + bool accepted_all_statuses = true; + for (const char* status : accepted_statuses) { + MyHGM->update_aws_aurora_bgd_status(300, status); + accepted_all_statuses = accepted_all_statuses && hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == status; + } + ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); + MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + + SQLite3_result* unrelated_reload = make_candidate(); + add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); + TestAuroraBGDRuntime::reload(MyHGM, unrelated_reload); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "configuration reload preserves the runtime BGD status"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 302 && green_reader == 312, + "configuration reload updates runtime green hostgroups"); + + MyHGM->update_aws_aurora_bgd_status(300, "NOT_A_BGD_STATUS"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "invalid Aurora BGD status leaves runtime state unchanged"); + + SQLite3_result* concurrent_reload = make_candidate(); + add_candidate_row(concurrent_reload, 300, 310, "303", "313", true, "concurrent"); + std::thread reload_thread([concurrent_reload]() { + TestAuroraBGDRuntime::reload(MyHGM, concurrent_reload); + }); + std::thread status_thread([]() { + MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); + }); + reload_thread.join(); + status_thread.join(); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "SWITCHOVER_IN_PROGRESS", "reload and status publication serialize without losing state"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 303 && green_reader == 313, + "serialized reload publishes its configured values"); + + SQLite3_result* empty_reload = make_candidate(); + TestAuroraBGDRuntime::reload(MyHGM, empty_reload); + ok(hgm_query_int( + "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == 0, "removing an Aurora deployment removes its runtime status row"); + ok(!TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader), + "removing an Aurora deployment removes its runtime info"); +} + int main() { - plan(30); + plan(44); test_init_minimal(); test_schema_contract(); // 12 @@ -237,6 +371,10 @@ int main() { test_legacy_projection_normalization(); // 3 test_invalid_replacement_removes_previous_row(); // 3 + ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 + test_runtime_ownership_and_status(); // 13 + test_cleanup_hostgroups(); + test_cleanup_minimal(); return exit_status(); } From e670d32777847e7a7807961b96c5b05828aec786 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:16:10 +0000 Subject: [PATCH 11/26] feat: persist Aurora BGD configuration fields --- include/proxysql_admin.h | 1 + lib/ProxySQL_Admin.cpp | 14 +++- lib/ProxySQL_Config.cpp | 52 ++++++++---- .../tests/unit/admin_disk_upgrade_unit-t.cpp | 84 ++++++++++++++++++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 55 +++++++++++- test/tap/tests/unit/config_write_unit-t.cpp | 66 ++++++++++++++- 6 files changed, 252 insertions(+), 20 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index d642b2756e..5529cb0d62 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -322,6 +322,7 @@ struct FlushVariableStats { class ProxySQL_Admin { friend class TestDiskUpgrade; + friend class TestAuroraBGDRuntime; private: volatile int main_shutdown; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 087b786d28..f73dae3178 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -189,6 +189,11 @@ static unordered_map&> module_tablenames = { { "pgsql_query_rules", pgsql_query_rules_tablenames }, }; +static const string mysql_aws_aurora_config_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, const string& p2, const string& p3) { string query; for (auto it = tbs.begin(); it != tbs.end(); it++) { @@ -197,7 +202,14 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con db->execute(query.c_str()); } if (p2 != "" && p3 != "") { - query = p2 + *it + p3 + *it; + const size_t wildcard_pos = p3.find('*'); + if (*it == "mysql_aws_aurora_hostgroups" && wildcard_pos != string::npos) { + string projected_select = p3; + projected_select.replace(wildcard_pos, 1, mysql_aws_aurora_config_columns); + query = p2 + *it + " (" + mysql_aws_aurora_config_columns + ")" + projected_select + *it; + } else { + query = p2 + *it + p3 + *it; + } db->execute(query.c_str()); } } diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 560e70c4cf..b4c7e910fb 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1282,7 +1282,10 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { sqlite_resultset = NULL; } - query=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups"; + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups"; admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); if (error) { // tolerate missing table (e.g. partial schemas in unit tests or old DBs) @@ -1298,18 +1301,21 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { data += "\t{\n"; addField(data, "writer_hostgroup", r->fields[0], ""); addField(data, "reader_hostgroup", r->fields[1], ""); - addField(data, "active", r->fields[2], ""); - addField(data, "aurora_port", r->fields[3], ""); - addField(data, "domain_name", r->fields[4]); - addField(data, "max_lag_ms", r->fields[5], ""); - addField(data, "check_interval_ms", r->fields[6], ""); - addField(data, "check_timeout_ms", r->fields[7], ""); - addField(data, "writer_is_also_reader", r->fields[8], ""); - addField(data, "new_reader_weight", r->fields[9], ""); - addField(data, "add_lag_ms", r->fields[10], ""); - addField(data, "min_lag_ms", r->fields[11], ""); - addField(data, "lag_num_checks", r->fields[12], ""); - addField(data, "comment", r->fields[13]); + addField(data, "green_writer_hostgroup", r->fields[2], ""); + addField(data, "green_reader_hostgroup", r->fields[3], ""); + addField(data, "active", r->fields[4], ""); + addField(data, "aurora_port", r->fields[5], ""); + addField(data, "domain_name", r->fields[6]); + addField(data, "max_lag_ms", r->fields[7], ""); + addField(data, "check_interval_ms", r->fields[8], ""); + addField(data, "check_timeout_ms", r->fields[9], ""); + addField(data, "writer_is_also_reader", r->fields[10], ""); + addField(data, "new_reader_weight", r->fields[11], ""); + addField(data, "add_lag_ms", r->fields[12], ""); + addField(data, "min_lag_ms", r->fields[13], ""); + addField(data, "lag_num_checks", r->fields[14], ""); + addField(data, "autopurge_missing_checks", r->fields[15], ""); + addField(data, "comment", r->fields[16]); data += "\t}"; isNext = true; @@ -1727,11 +1733,14 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { if (root.exists("mysql_aws_aurora_hostgroups")==true) { const Setting &mysql_aws_aurora_hostgroups = root["mysql_aws_aurora_hostgroups"]; int count = mysql_aws_aurora_hostgroups.getLength(); - char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + // Green hostgroups are nullable -> passed as %s ("NULL" or an integer). + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; for (i=0; i< count; i++) { const Setting &line = mysql_aws_aurora_hostgroups[i]; int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int active=1; // default int aurora_port; int max_lag_ms; @@ -1753,6 +1762,19 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { proxy_error("Admin: detected a mysql_aws_aurora_hostgroups in config file without a mandatory reader_hostgroup\n"); continue; } + std::string green_writer_str; + std::string green_reader_str; + if (line.lookupValue("green_writer_hostgroup", green_writer_hostgroup)==false) { + green_writer_str = "NULL"; + } else { + green_writer_str = std::to_string(green_writer_hostgroup); + } + if (line.lookupValue("green_reader_hostgroup", green_reader_hostgroup)==false) { + green_reader_str = "NULL"; + } else { + green_reader_str = std::to_string(green_reader_hostgroup); + } + if (line.lookupValue("active", active)==false) active=1; if (line.lookupValue("aurora_port", aurora_port)==false) aurora_port=3306; if (line.lookupValue("max_lag_ms", max_lag_ms)==false) max_lag_ms=600000; if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; @@ -1776,7 +1798,7 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { const size_t safe_domain_len = safe_strlen(safe_domain); const size_t query_len = query_base_len + safe_comment_len + safe_domain_len + 256; // 128 vs sizeof(int)*8 char *query=(char *)l_alloc(query_len); - format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); + format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, green_writer_str.c_str(), green_reader_str.c_str(), active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); //fprintf(stderr, "%s\n", query); admindb->execute(query); if (o!=o1) free(o); diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index d223105327..1ac5f824dd 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -80,6 +80,9 @@ class TestDiskUpgrade { void upgrade_rest_api_routes() { admin->disk_upgrade_rest_api_routes(); } void upgrade_mysql_query_rules() { admin->disk_upgrade_mysql_query_rules(); } void upgrade_pgsql_replication_hostgroups() { admin->disk_upgrade_pgsql_replication_hostgroups(); } + void use_config_db_as_admin_db() { admin->admindb = admin->configdb; } + void save_mysql_servers_to_disk() { admin->__insert_or_replace_disktable_select_maintable(); } + void load_mysql_servers_from_disk() { admin->__insert_or_replace_maintable_select_disktable(); } }; // --------------------------------------------------------------------------- @@ -575,12 +578,90 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { ok(w2 == 500, "mysql_servers: normal weight preserved (got %d)", w2); } +static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { + TestDiskUpgrade t; + SQLite3DB *db = t.db(); + t.use_config_db_as_admin_db(); + + struct TableDefinition { + const char *name; + const char *definition; + }; + const TableDefinition tables[] = { + { "mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS }, + { "mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS }, + { "mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS }, + { "mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS }, + { "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS }, + { "mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS }, + { "mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES }, + { "mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS }, + { "mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES }, + { "mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING }, + { "mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS }, + { "mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS }, + { "mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES }, + { "mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS }, + { "global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES }, + { "scheduler", ADMIN_SQLITE_TABLE_SCHEDULER }, + { "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES }, + { "proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS }, + { "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS }, + { "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS }, + { "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES }, + { "pgsql_servers_ssl_params", ADMIN_SQLITE_TABLE_PGSQL_SERVERS_SSL_PARAMS }, + { "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES }, + { "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING }, + { "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS }, + { "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS }, + { "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES }, + { "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS } + }; + for (const TableDefinition& table : tables) { + db->execute(table.definition); + } + db->execute("ATTACH DATABASE ':memory:' AS disk"); + for (const TableDefinition& table : tables) { + std::string query = std::string("CREATE TABLE disk.") + table.name + + " AS SELECT * FROM main." + table.name + " WHERE 0"; + db->execute(query.c_str()); + } + + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES (600,601,602,603,'.disk.example','disk round trip')" + ); + // Model a node-local runtime field on the source. The disk projection must ignore it. + db->execute("ALTER TABLE main.mysql_aws_aurora_hostgroups ADD COLUMN bgd_status VARCHAR NOT NULL DEFAULT 'LOCAL'"); + db->execute("UPDATE main.mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS'"); + + t.save_mysql_servers_to_disk(); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk SAVE preserves both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, + "Aurora disk SAVE excludes node-local bgd_status"); + + db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); + t.load_mysql_servers_from_disk(); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk LOAD restores both configured green hostgroups"); + ok(query_string(db, + "SELECT bgd_status FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "LOCAL", + "Aurora disk LOAD leaves node-local bgd_status at its local default"); +} + // ============================================================================ // main // ============================================================================ int main() { - plan(67); + plan(71); test_init_minimal(); // scheduler tests @@ -615,6 +696,7 @@ int main() { // Multi-row tests test_scheduler_upgrade_preserves_multiple_rows(); test_mysql_servers_upgrade_multiple_rows_with_fixes(); + test_aurora_hostgroups_disk_roundtrip_uses_configured_projection(); test_cleanup_minimal(); return exit_status(); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index e4adee5c4f..104992bf75 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -44,6 +44,38 @@ class TestAuroraBGDRuntime { pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); return true; } + + static SQLite3DB* materialize_aurora_table(bool runtime) { + void* memory = calloc(1, sizeof(ProxySQL_Admin)); + ProxySQL_Admin* admin = reinterpret_cast(memory); + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + admin->admindb = db; + + if (runtime) { + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); + } else { + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + } + + admin->save_mysql_servers_runtime_to_database(runtime); + free(admin); + return db; + } }; static std::string query_string(SQLite3DB* db, const char* query) { @@ -319,6 +351,25 @@ static void test_runtime_ownership_and_status() { } ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + SQLite3DB* configured_db = TestAuroraBGDRuntime::materialize_aurora_table(false); + ok(query_string(configured_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "SAVE from runtime preserves both configured green hostgroups"); + ok(query_int(configured_db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "SAVE from runtime excludes bgd_status from configuration"); + delete configured_db; + + SQLite3DB* runtime_db = TestAuroraBGDRuntime::materialize_aurora_table(true); + ok(query_string(runtime_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "runtime materialization preserves both configured green hostgroups"); + ok(query_string(runtime_db, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", + "runtime materialization includes the node-local bgd_status"); + delete runtime_db; SQLite3_result* unrelated_reload = make_candidate(); add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); @@ -362,7 +413,7 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(44); + plan(48); test_init_minimal(); test_schema_contract(); // 12 @@ -372,7 +423,7 @@ int main() { test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 13 + test_runtime_ownership_and_status(); // 17 test_cleanup_hostgroups(); test_cleanup_minimal(); diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index cda1fac9a0..1a12a85a7d 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -470,6 +470,69 @@ static void test_write_mysql_servers_replication_hostgroups() { delete db; } +static void test_roundtrip_mysql_aws_aurora_hostgroups() { + SQLite3DB* db = create_test_db(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (500,501,502,503,0,3307,'.bgd.example',321,1200,900,1,7,40,20,3,9,'aurora bgd')" + ); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES (510,511,NULL,NULL,'.legacy.example','ordinary aurora')" + ); + + ProxySQL_Config cfg(db); + std::string data; + int write_rc = cfg.Write_MySQL_Servers_to_configfile(data); + + ok(write_rc == 0, "RT Aurora hostgroups: configuration export succeeds"); + ok(data.find("green_writer_hostgroup=502") != std::string::npos, + "RT Aurora hostgroups: green writer is exported"); + ok(data.find("green_reader_hostgroup=503") != std::string::npos, + "RT Aurora hostgroups: green reader is exported"); + ok(data.find("active=0") != std::string::npos && + data.find("autopurge_missing_checks=9") != std::string::npos, + "RT Aurora hostgroups: later configured columns retain their values"); + ok(data.find("bgd_status") == std::string::npos, + "RT Aurora hostgroups: runtime status is absent from configuration export"); + + db->execute("DELETE FROM mysql_aws_aurora_hostgroups"); + ProxySQL_ConfigFile* cf = load_config_from_string(data); + ProxySQL_ConfigFile* saved = GloVars.confFile; + GloVars.confFile = cf; + std::string error; + int rows = cfg.Read_MySQL_Servers_from_configfile(error); + GloVars.confFile = saved; + delete cf; + + ok(rows == 2, "RT Aurora hostgroups: configuration import restores both rows (got %d)", rows); + ok(db_select_string(db, "green_writer_hostgroup || ',' || green_reader_hostgroup", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "502,503", + "RT Aurora hostgroups: both green hostgroups survive the round trip"); + ok(db_select_string(db, "green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=510") == "1", + "RT Aurora hostgroups: omitted green hostgroups remain NULL"); + ok(db_select_string(db, "active", "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "0", + "RT Aurora hostgroups: inactive configuration survives the round trip"); + ok(db_select_string(db, "autopurge_missing_checks", "mysql_aws_aurora_hostgroups", + "writer_hostgroup=500") == "9", + "RT Aurora hostgroups: autopurge configuration survives the round trip"); + + delete db; +} + // ============================================================ // Write_Global_Variables_to_configfile() // ============================================================ @@ -1011,7 +1074,7 @@ static void test_roundtrip_pgsql_firewall() { // ============================================================ int main() { - plan(161); // matches exact number of ok() assertions in this file + plan(171); // matches exact number of ok() assertions in this file test_init_minimal(); // MySQL side - existing + new data-driven tests @@ -1040,6 +1103,7 @@ int main() { test_write_mysql_servers_empty(); test_write_mysql_servers_with_data(); test_write_mysql_servers_replication_hostgroups(); + test_roundtrip_mysql_aws_aurora_hostgroups(); test_write_global_variables_empty(); test_write_global_variables_single_prefix(); From d085604bcceca3f269566abe6cb22f20a9080fe0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:45:06 +0000 Subject: [PATCH 12/26] feat: synchronize Aurora BGD configuration --- include/ProxySQL_Cluster.hpp | 2 +- lib/Admin_Handler.cpp | 6 +- lib/ProxySQL_Cluster.cpp | 81 ++++++---- test/tap/tests/test_cluster_sync-t.cpp | 197 ++++++++++++++++--------- 4 files changed, 183 insertions(+), 103 deletions(-) diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index eaccc4bbb6..6f1e07fb6c 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -70,7 +70,7 @@ #define CLUSTER_QUERY_PGSQL_SERVERS_SSL_PARAMS "PROXY_SELECT hostname, port, username, ssl_ca, ssl_cert, ssl_key, ssl_crl, ssl_crlpath, ssl_protocol_version_range, comment FROM runtime_pgsql_servers_ssl_params ORDER BY hostname, port, username" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ -#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" +#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_rds_bgd_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE auto_generated=0 ORDER BY writer_hostgroup" diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 7aaa379608..0566703047 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -4501,7 +4501,11 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { (query_no_space_length==sizeof("CHECKSUM MEM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MEM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length)) || (query_no_space_length==sizeof("CHECKSUM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length))){ - char *q=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; + char *q=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index f82e571f45..bc223ec646 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -2389,41 +2389,62 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_aurora_hostgroups table\n"); proxy_info("Cluster: Writing mysql_aws_aurora_hostgroups table\n"); GloAdmin->admindb->execute(SQLQueries::DELETE_MYSQL_AWS_AURORA_HOSTGROUPS); - q = (char*)"INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment) "; - while ((row = mysql_fetch_row(results[4]))) { - int l = 0; - for (int i = 0; i < 14; i++) { - l += strlen(row[i]); - } - char* o = nullptr; - char* query = nullptr; - std::string fqs = q; - - if (row[14] != nullptr) { - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, '%s')"; - o = escape_string_single_quotes(row[14], false); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen(o) + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); - // free in case of 'o' being a copy - if (o != row[14]) { - free(o); + { + const char* insert_aurora = + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = GloAdmin->admindb->prepare_v2(insert_aurora); + ASSERT_SQLITE_OK(prepare_rc, GloAdmin->admindb); + sqlite3_stmt* statement = statement_unique.get(); + + while ((row = mysql_fetch_row(results[4]))) { + int rc=(*proxy_sqlite3_bind_int64)(statement, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atol(row[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); } - } else { - // In case of comment being null, placeholder must not have '' - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; - o = const_cast("NULL"); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen("NULL") + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atol(row[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atol(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, row[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, atol(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atol(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11, atol(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, atol(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13, atol(row[12])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14, atol(row[13])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, atol(row[14])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, atol(row[15])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[16]) { + rc=(*proxy_sqlite3_bind_text)(statement, 17, row[16], -1, SQLITE_TRANSIENT); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 17); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); } - - GloAdmin->admindb->execute(query); - free(query); } proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement((char*) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 10b82af49d..1e85d03cd7 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -217,6 +217,25 @@ const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; using mysql_server_tuple = tuple; +struct aurora_hostgroups_row_t { + int writer_hostgroup; + int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; + int active; + int aurora_port; + const char* domain_name; + int max_lag_ms; + int check_interval_ms; + int check_timeout_ms; + int writer_is_also_reader; + int new_reader_weight; + int add_lag_ms; + int min_lag_ms; + int lag_num_checks; + const char* comment; +}; + int check_mysql_servers_sync( const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_admin, @@ -1213,7 +1232,7 @@ int main(int, char**) { plan( // Sync tests by values - 16 + + 17 + // Module checkums tests; enabled and disabled checksums check_modules_checksums_sync__tests + (cl.use_noise ? 3 : 0) @@ -2138,14 +2157,15 @@ int main(int, char**) { // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1), - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1), + "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; + std::vector insert_aws_aurora_values { + { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, nullptr }, + { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, nullptr }, + { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, nullptr }, + { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, nullptr }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2154,26 +2174,29 @@ int main(int, char**) { string_format( t_insert_mysql_aws_aurora_hostgroups, insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks ); insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); } const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS NULL"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; @@ -2183,19 +2206,21 @@ int main(int, char**) { string_format( t_select_aws_aurora_inserted_entries, select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks ); select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); } @@ -2265,14 +2290,15 @@ int main(int, char**) { // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), + "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + std::vector insert_aws_aurora_values { + { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2281,27 +2307,30 @@ int main(int, char**) { string_format( t_insert_mysql_aws_aurora_hostgroups, insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); } const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment='%s'"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; @@ -2311,20 +2340,22 @@ int main(int, char**) { string_format( t_select_aws_aurora_inserted_entries, select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); } @@ -2371,6 +2402,30 @@ int main(int, char**) { system(print_replica_aws_aurora_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); + MYSQL_QUERY__(proxy_admin, + "CREATE VIEW aurora_bgd_local_status AS " + "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); + MYSQL_QUERY__(r_proxy_admin, + "CREATE VIEW aurora_bgd_local_status AS " + "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); + MYSQL_QUERY__(proxy_admin, + "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='AVAILABLE' WHERE writer_hostgroup=1000"); + MYSQL_QUERY__(r_proxy_admin, + "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS' WHERE writer_hostgroup=1000"); + sleep(2); + MYSQL_QUERY__(proxy_admin, + "SELECT COUNT(*) FROM aurora_bgd_local_status " + "WHERE writer_hostgroup=1000 AND bgd_status='AVAILABLE'"); + const bool master_status_is_local = fetch_single_int_res(proxy_admin) == 1; + MYSQL_QUERY__(r_proxy_admin, + "SELECT COUNT(*) FROM aurora_bgd_local_status " + "WHERE writer_hostgroup=1000 AND bgd_status='SWITCHOVER_IN_PROGRESS'"); + const bool replica_status_is_local = fetch_single_int_res(r_proxy_admin) == 1; + ok(master_status_is_local && replica_status_is_local, + "Aurora cluster sync leaves each node's bgd_status unchanged."); + MYSQL_QUERY__(proxy_admin, "DROP VIEW aurora_bgd_local_status"); + MYSQL_QUERY__(r_proxy_admin, "DROP VIEW aurora_bgd_local_status"); + // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); MYSQL_QUERY__(proxy_admin, "INSERT INTO mysql_aws_aurora_hostgroups SELECT * FROM mysql_aws_aurora_hostgroups_sync_test_2687"); From 8e67581fb13ae8b9a3227e2c2c5929f13a5e8faa Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:59:15 +0000 Subject: [PATCH 13/26] fix: exclude Aurora BGD status from cluster checksum --- lib/MySQL_HostGroups_Manager.cpp | 11 ++++++++++- test/tap/groups/groups.json | 1 + test/tap/tests/unit/aurora_bgd_config_unit-t.cpp | 7 +++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 1a1c0b01c5..468465f1a0 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1151,7 +1151,16 @@ void MySQL_HostGroups_Manager::CUCFT1( int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + string query; + if (TableName == "mysql_aws_aurora_hostgroups") { + query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups ORDER BY " + ColumnName; + } else { + query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + } mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); if (resultset) { if (resultset->rows_count) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index cb972d6154..897b4fdc88 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -11,6 +11,7 @@ "ai_error_handling_edge_cases-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_llm_retry_scenarios-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_validation-t" : [ "ai-g1","@proxysql_min_version:4.0" ], + "aurora_bgd_config_unit-t" : [ "unit-tests-g1" ], "auth_unit-t" : [ "unit-tests-g1" ], "backend_sync_unit-t" : [ "unit-tests-g1" ], "basic-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index 104992bf75..b1600a4215 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -325,6 +325,7 @@ static void test_runtime_ownership_and_status() { "SELECT GROUP_CONCAT(bgd_status, ',') FROM " "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); + const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); @@ -334,6 +335,8 @@ static void test_runtime_ownership_and_status() { ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); + ok(MyHGM->gen_global_mysql_servers_v2_checksum(0) == checksum_before_status, + "node-local Aurora BGD status is excluded from the cluster checksum"); const char* accepted_statuses[] = { "NONE", "AVAILABLE", @@ -413,7 +416,7 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(48); + plan(49); test_init_minimal(); test_schema_contract(); // 12 @@ -423,7 +426,7 @@ int main() { test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 17 + test_runtime_ownership_and_status(); // 18 test_cleanup_hostgroups(); test_cleanup_minimal(); From 7a7303575cb26a0cb78a75b944b903135f7bc193 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 21:42:25 +0000 Subject: [PATCH 14/26] refactor: simplify Aurora BGD configuration paths --- include/MySQL_HostGroups_Manager.h | 7 +- include/proxysql_admin.h | 20 +++- lib/MySQL_HostGroups_Manager.cpp | 82 ++++--------- lib/ProxySQL_Admin.cpp | 185 ++++++++++++++++++----------- 4 files changed, 162 insertions(+), 132 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 4d1c56aafd..a340e6fc10 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -388,9 +388,9 @@ class AWS_Aurora_Info { /** * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. * - * The returned result contains only valid rows, ordered like the Aurora Admin - * table. Legacy projections without green hostgroups are normalized with both - * green values set to SQL NULL. The caller owns the returned result. + * The candidate must exactly match the configured Aurora Admin table projection. + * The returned result contains only valid rows in that canonical order. The + * caller owns the returned result. */ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( const SQLite3_result* candidate, @@ -998,6 +998,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @brief Creates a resultset with the current full content of the target table. * @param string The target table. Valid values are: * - "mysql_aws_aurora_hostgroups" + * - "runtime_mysql_aws_aurora_hostgroups" * - "mysql_galera_hostgroups" * - "mysql_group_replication_hostgroups" * - "mysql_replication_hostgroups" diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5529cb0d62..4907c3245e 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -320,9 +320,27 @@ struct FlushVariableStats { int unknown = 0; }; +/** + * @brief Replace an Admin Aurora hostgroup table with an HGM runtime snapshot. + * + * @param db Destination Admin database. + * @param resultset Aurora rows returned by MySQL_HostGroups_Manager. + * @param runtime Selects runtime_mysql_aws_aurora_hostgroups when true and + * mysql_aws_aurora_hostgroups when false. + * @return true when the destination table was replaced. + */ +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +); + +/** Copy the configured Aurora hostgroup projection between Admin and disk. */ +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db); +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db); + class ProxySQL_Admin { friend class TestDiskUpgrade; - friend class TestAuroraBGDRuntime; private: volatile int main_shutdown; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 468465f1a0..27ce3332ec 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -9,7 +9,7 @@ using json = nlohmann::json; #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" -#include +#include #include #include #include @@ -98,15 +98,6 @@ struct Aurora_Config_Row { bool locally_valid; }; -int aws_aurora_column_index(const SQLite3_result* candidate, const char* name) { - for (size_t i = 0; i < candidate->column_definition.size(); ++i) { - if (strcasecmp(candidate->column_definition[i]->name, name) == 0) { - return static_cast(i); - } - } - return -1; -} - std::string aws_aurora_nullable_value(const char* value) { return value ? value : "NULL"; } @@ -126,35 +117,33 @@ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( return filtered; } - std::vector source_indexes(AWS_AURORA_CONFIG_COLUMN_COUNT, -1); - for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - source_indexes[i] = aws_aurora_column_index(candidate, AWS_AURORA_CONFIG_COLUMNS[i]); - } - - // Older peers send the pre-BGD 15-column projection. Treat the two absent - // green fields as SQL NULL until cluster synchronization is upgraded. - const bool green_writer_absent = source_indexes[2] == -1; - const bool green_reader_absent = source_indexes[3] == -1; - if (green_writer_absent != green_reader_absent) { + if (candidate->columns != static_cast(candidate->column_definition.size())) { errors.emplace_back( - "mysql_aws_aurora_hostgroups rejected: candidate projection contains only one green hostgroup column" + "mysql_aws_aurora_hostgroups rejected: candidate projection metadata is inconsistent" ); return filtered; } - for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - if ((i == 2 || i == 3) && green_writer_absent) { - continue; - } - if (source_indexes[i] == -1) { - errors.emplace_back( - std::string("mysql_aws_aurora_hostgroups rejected: candidate projection is missing ") + - AWS_AURORA_CONFIG_COLUMNS[i] - ); + const int comparable_columns = std::min(candidate->columns, AWS_AURORA_CONFIG_COLUMN_COUNT); + for (int i = 0; i < comparable_columns; ++i) { + const char* actual_name = candidate->column_definition[i]->name; + if (strcasecmp(actual_name, AWS_AURORA_CONFIG_COLUMNS[i]) != 0) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups rejected: candidate projection column " << i + << " must be " << AWS_AURORA_CONFIG_COLUMNS[i] + << ", got " << actual_name; + errors.emplace_back(message.str()); return filtered; } } + if (candidate->columns != AWS_AURORA_CONFIG_COLUMN_COUNT) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection must contain exactly 17 columns" + ); + return filtered; + } + std::vector rows; rows.reserve(candidate->rows.size()); @@ -162,9 +151,7 @@ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( Aurora_Config_Row row; row.fields.resize(AWS_AURORA_CONFIG_COLUMN_COUNT, nullptr); for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - if (source_indexes[i] != -1) { - row.fields[i] = source_row->fields[source_indexes[i]]; - } + row.fields[i] = source_row->fields[i]; } row.writer_hostgroup = row.fields[0] ? atoi(row.fields[0]) : -1; @@ -2460,6 +2447,9 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { char * query = (char *)""; if (name == "mysql_aws_aurora_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "runtime_mysql_aws_aurora_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { @@ -4190,24 +4180,6 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write } void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { - static const std::array valid_statuses { - "NONE", - "AVAILABLE", - "SWITCHOVER_INITIATED", - "SWITCHOVER_IN_PROGRESS", - "SWITCHOVER_IN_POST_PROCESSING", - "SWITCHOVER_COMPLETED" - }; - - bool valid = std::any_of(valid_statuses.begin(), valid_statuses.end(), [&bgd_status](const char* status) { - return bgd_status == status; - }); - if (!valid) { - proxy_error("Invalid AWS Aurora BGD runtime status '%s' for writer hostgroup %d\n", - bgd_status.c_str(), writer_hostgroup); - return; - } - wrlock(); const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; auto [prepare_rc, statement_unique] = mydb->prepare_v2(query); @@ -6849,11 +6821,9 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { // it is now time to build a new structure in Monitor - if (GloMyMon) { - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - update_aws_aurora_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - } + pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); + update_aws_aurora_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f73dae3178..9c54baf9cd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -143,7 +143,6 @@ static const vector mysql_servers_tablenames = { "mysql_replication_hostgroups", "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", - "mysql_aws_aurora_hostgroups", "mysql_aws_rds_bgd_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", @@ -202,19 +201,33 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con db->execute(query.c_str()); } if (p2 != "" && p3 != "") { - const size_t wildcard_pos = p3.find('*'); - if (*it == "mysql_aws_aurora_hostgroups" && wildcard_pos != string::npos) { - string projected_select = p3; - projected_select.replace(wildcard_pos, 1, mysql_aws_aurora_config_columns); - query = p2 + *it + " (" + mysql_aws_aurora_config_columns + ")" + projected_select + *it; - } else { - query = p2 + *it + p3 + *it; - } + query = p2 + *it + p3 + *it; db->execute(query.c_str()); } } } +static bool copy_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const string& destination_schema, + const string& source_schema +) { + const string table_name = "mysql_aws_aurora_hostgroups"; + const string query = + "INSERT OR REPLACE INTO " + destination_schema + "." + table_name + + " (" + mysql_aws_aurora_config_columns + ") SELECT " + + mysql_aws_aurora_config_columns + " FROM " + source_schema + "." + table_name; + return db->execute(query.c_str()); +} + +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "main", "disk"); +} + +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "disk", "main"); +} + static int round_intv_to_time_interval(const char* name, int _intv) { int intv = _intv; @@ -5772,6 +5785,7 @@ int ProxySQL_Admin::flush_debug_levels_database_to_runtime(SQLite3DB *db) { void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("PRAGMA foreign_keys = OFF"); BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); @@ -5856,6 +5870,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.mysql_users SELECT * FROM main.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); @@ -5964,8 +5979,16 @@ void ProxySQL_Admin::flush_GENERIC__from_to(const string& name, const string& di assert(it != module_tablenames.end()); if (direction == "disk_to_memory") { BQE1(admindb, it->second, "DELETE FROM main.", "INSERT INTO main.", " SELECT * FROM disk."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); + } } else if (direction == "memory_to_disk") { BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM main."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM disk.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); + } } else { assert(0); } @@ -7323,6 +7346,83 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { free(args); } +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +) { + if (db == nullptr) { + return false; + } + + const int expected_columns = runtime ? 18 : 17; + if (resultset != nullptr && resultset->columns < expected_columns) { + proxy_error( + "Cannot materialize mysql_aws_aurora_hostgroups: expected at least %d columns, got %d\n", + expected_columns, + resultset->columns + ); + return false; + } + + const char* delete_query = runtime + ? "DELETE FROM main.runtime_mysql_aws_aurora_hostgroups" + : "DELETE FROM main.mysql_aws_aurora_hostgroups"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", delete_query); + if (!db->execute(delete_query)) { + return false; + } + if (resultset == nullptr) { + return true; + } + + const char* insert_query = runtime + ? "INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)" + : "INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = db->prepare_v2(insert_query); + ASSERT_SQLITE_OK(prepare_rc, db); + sqlite3_stmt* statement = statement_unique.get(); + + for (const SQLite3_row* row : resultset->rows) { + int rc = (*proxy_sqlite3_bind_int64)(statement, 1, atoi(row->fields[0])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atoi(row->fields[1])); ASSERT_SQLITE_OK(rc, db); + if (row->fields[2]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atoi(row->fields[2])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, db); + if (row->fields[3]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 4, atoi(row->fields[3])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 5, atoi(row->fields[4])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 6, atoi(row->fields[5])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 7, row->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 8, atoi(row->fields[7])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 9, atoi(row->fields[8])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 10, atoi(row->fields[9])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 11, atoi(row->fields[10])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 12, atoi(row->fields[11])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 13, atoi(row->fields[12])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 14, atoi(row->fields[13])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 15, atoi(row->fields[14])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 16, atoi(row->fields[15])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 17, row->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + if (runtime) { + rc = (*proxy_sqlite3_bind_text)(statement, 18, row->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + } + + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, db); + } + + return true; +} + void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { // make sure that the caller has called mysql_servers_wrlock() char *query=NULL; @@ -7536,69 +7636,10 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { resultset = NULL; // dump mysql_aws_aurora_hostgroups - - if (_runtime) { - query=(char *)"DELETE FROM main.runtime_mysql_aws_aurora_hostgroups"; - } else { - query=(char *)"DELETE FROM main.mysql_aws_aurora_hostgroups"; - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute(query); - resultset=MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); - if (resultset) { - int rc; - sqlite3_stmt *statement=NULL; - - char *query=NULL; - if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; - } else { - query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; - } - - auto [rc1, statement_unique] = admindb->prepare_v2(query); - rc = rc1; - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, admindb); - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); - if (r->fields[2]) { - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); - } else { - rc=(*proxy_sqlite3_bind_null)(statement, 3); - } - ASSERT_SQLITE_OK(rc, admindb); - if (r->fields[3]) { - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); - } else { - rc=(*proxy_sqlite3_bind_null)(statement, 4); - } - ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, atoi(r->fields[10])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, atoi(r->fields[12])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, atoi(r->fields[13])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 15, atoi(r->fields[14])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 16, atoi(r->fields[15])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - if (_runtime) { - rc=(*proxy_sqlite3_bind_text)(statement, 18, r->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - } - - SAFE_SQLITE3_STEP2(statement); - rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); - } - } + resultset=MyHGM->dump_table_mysql( + _runtime ? "runtime_mysql_aws_aurora_hostgroups" : "mysql_aws_aurora_hostgroups" + ); + materialize_mysql_aws_aurora_hostgroups(admindb, resultset, _runtime); if(resultset) delete resultset; resultset=NULL; From 3d9c54a799faff8bae8a8e3cf57259a3ed4e0045 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 21:42:29 +0000 Subject: [PATCH 15/26] test: streamline Aurora BGD phase 1 coverage --- test/tap/test_helpers/test_init.cpp | 20 ++ test/tap/test_helpers/test_init.h | 13 ++ test/tap/tests/test_cluster_sync-t.cpp | 211 ++---------------- .../tests/unit/admin_disk_upgrade_unit-t.cpp | 109 ++++----- .../tests/unit/aurora_bgd_config_unit-t.cpp | 166 +++++++------- 5 files changed, 172 insertions(+), 347 deletions(-) diff --git a/test/tap/test_helpers/test_init.cpp b/test/tap/test_helpers/test_init.cpp index 08261b2efd..57192d5b00 100644 --- a/test/tap/test_helpers/test_init.cpp +++ b/test/tap/test_helpers/test_init.cpp @@ -20,6 +20,7 @@ #include "PgSQL_Query_Cache.h" #include "MySQL_Query_Processor.h" #include "PgSQL_Query_Processor.h" +#include "MySQL_Monitor.hpp" #include "test_globals.h" #include "test_init.h" @@ -32,6 +33,7 @@ extern MySQL_Query_Cache *GloMyQC; extern PgSQL_Query_Cache *GloPgQC; extern MySQL_Query_Processor *GloMyQPro; extern PgSQL_Query_Processor *GloPgQPro; +extern MySQL_Monitor *GloMyMon; // GloMTH is declared extern in proxysql_utils.h. // GloPTH has no extern declaration in any header, so we add one here. @@ -205,3 +207,21 @@ void test_cleanup_hostgroups() { PgHGM = nullptr; } } + +// ============================================================================ +// MySQL Monitor +// ============================================================================ + +int test_init_monitor() { + if (GloMyMon == nullptr) { + GloMyMon = new MySQL_Monitor(); + } + return 0; +} + +void test_cleanup_monitor() { + if (GloMyMon != nullptr) { + delete GloMyMon; + GloMyMon = nullptr; + } +} diff --git a/test/tap/test_helpers/test_init.h b/test/tap/test_helpers/test_init.h index 5818bde0ba..d623d06152 100644 --- a/test/tap/test_helpers/test_init.h +++ b/test/tap/test_helpers/test_init.h @@ -128,4 +128,17 @@ int test_init_hostgroups(); */ void test_cleanup_hostgroups(); +/** + * @brief Initialize the MySQL monitor without starting its worker threads. + * + * @pre test_init_minimal() must have been called. + * @return 0 on success, non-zero on failure. + */ +int test_init_monitor(); + +/** + * @brief Clean up resources allocated by test_init_monitor(). + */ +void test_cleanup_monitor(); + #endif /* TEST_INIT_H */ diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 1e85d03cd7..f39fb62ef4 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -6,7 +6,7 @@ * - 'mysql_galera_hostgroups' with and without NULL comments. * - 'mysql_group_replication_hostgroups' with and without NULL comments. * - 'proxysql_servers' with new values and empty (exercising bug from '#3847'). - * - 'mysql_aws_aurora_hostgroups' with and without NULL comments. + * - 'mysql_aws_aurora_hostgroups' with configured and NULL green hostgroups/comments. * - 'mysql_variables'. * - 'admin_variables'. * @@ -220,8 +220,8 @@ using mysql_server_tuple = tuple insert_aws_aurora_values { - { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, nullptr }, - { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, nullptr }, - { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, nullptr }, - { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, nullptr }, - }; - std::vector insert_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string insert_aws_aurora_hostgroup_query = ""; - string_format( - t_insert_mysql_aws_aurora_hostgroups, - insert_aws_aurora_hostgroup_query, - values.writer_hostgroup, - values.reader_hostgroup, - values.green_writer_hostgroup, - values.green_reader_hostgroup, - values.active, - values.aurora_port, - values.domain_name, - values.max_lag_ms, - values.check_interval_ms, - values.check_timeout_ms, - values.writer_is_also_reader, - values.new_reader_weight, - values.add_lag_ms, - values.min_lag_ms, - values.lag_num_checks - ); - insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); - } - - const char* t_select_aws_aurora_inserted_entries = - "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " - "AND active=%d AND aurora_port=%d AND domain_name='%s' " - "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS NULL"; - std::vector select_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string select_aws_aurora_hostgroup_query = ""; - string_format( - t_select_aws_aurora_inserted_entries, - select_aws_aurora_hostgroup_query, - values.writer_hostgroup, - values.reader_hostgroup, - values.green_writer_hostgroup, - values.green_reader_hostgroup, - values.active, - values.aurora_port, - values.domain_name, - values.max_lag_ms, - values.check_interval_ms, - values.check_timeout_ms, - values.writer_is_also_reader, - values.new_reader_weight, - values.add_lag_ms, - values.min_lag_ms, - values.lag_num_checks - ); - select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); - } - - // SETUP CONFIG - - // Backup current table - MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - - // Insert the new aws_aurora hostgroups values - for (const auto& query : insert_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(proxy_admin, query.c_str()); - } - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; - system(print_master_aws_aurora_hostgroups.c_str()); - - // SYNCH CHECK - - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; - system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' with NULL comments should be synced."); - - // TEARDOWN CONFIG - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "INSERT INTO mysql_aws_aurora_hostgroups SELECT * FROM mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "DROP TABLE mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - } - - sleep(2); - - // Check 'mysql_aws_aurora_hostgroups' synchronization + // Check 'mysql_aws_aurora_hostgroups' synchronization for configured and NULL values { std::string print_master_aws_aurora_hostgroups = ""; string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); @@ -2293,12 +2160,10 @@ int main(int, char**) { "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " - "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + "VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %s)"; std::vector insert_aws_aurora_values { - { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1000, 1004, "1010", "1014", 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "'reader_writer_test_aws_aurora_hostgroup'" }, + { 1002, 1006, "NULL", "NULL", 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "NULL" }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2329,10 +2194,10 @@ int main(int, char**) { const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup IS %s AND green_reader_hostgroup IS %s " "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment='%s'"; + "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS %s"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; for (auto const& values : insert_aws_aurora_values) { @@ -2361,6 +2226,7 @@ int main(int, char**) { } // Backup current table + MYSQL_QUERY__(proxy_admin, "DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_sync_test_2687"); MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); @@ -2372,59 +2238,14 @@ int main(int, char**) { std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; system(print_master_aws_aurora_hostgroups.c_str()); - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } + const int wait_res = proc_wait_checks( + wait_for_conds(r_proxy_admin, select_mysql_aws_aurora_hostgroup_queries, SYNC_TIMEOUT) + ); std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); - - MYSQL_QUERY__(proxy_admin, - "CREATE VIEW aurora_bgd_local_status AS " - "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(r_proxy_admin, - "CREATE VIEW aurora_bgd_local_status AS " - "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, - "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='AVAILABLE' WHERE writer_hostgroup=1000"); - MYSQL_QUERY__(r_proxy_admin, - "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS' WHERE writer_hostgroup=1000"); - sleep(2); - MYSQL_QUERY__(proxy_admin, - "SELECT COUNT(*) FROM aurora_bgd_local_status " - "WHERE writer_hostgroup=1000 AND bgd_status='AVAILABLE'"); - const bool master_status_is_local = fetch_single_int_res(proxy_admin) == 1; - MYSQL_QUERY__(r_proxy_admin, - "SELECT COUNT(*) FROM aurora_bgd_local_status " - "WHERE writer_hostgroup=1000 AND bgd_status='SWITCHOVER_IN_PROGRESS'"); - const bool replica_status_is_local = fetch_single_int_res(r_proxy_admin) == 1; - ok(master_status_is_local && replica_status_is_local, - "Aurora cluster sync leaves each node's bgd_status unchanged."); - MYSQL_QUERY__(proxy_admin, "DROP VIEW aurora_bgd_local_status"); - MYSQL_QUERY__(r_proxy_admin, "DROP VIEW aurora_bgd_local_status"); + ok(wait_res == EXIT_SUCCESS, + "'mysql_aws_aurora_hostgroups' should sync configured and NULL values: '%d'", wait_res); // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index 1ac5f824dd..26d2d35cc8 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -80,9 +80,6 @@ class TestDiskUpgrade { void upgrade_rest_api_routes() { admin->disk_upgrade_rest_api_routes(); } void upgrade_mysql_query_rules() { admin->disk_upgrade_mysql_query_rules(); } void upgrade_pgsql_replication_hostgroups() { admin->disk_upgrade_pgsql_replication_hostgroups(); } - void use_config_db_as_admin_db() { admin->admindb = admin->configdb; } - void save_mysql_servers_to_disk() { admin->__insert_or_replace_disktable_select_maintable(); } - void load_mysql_servers_from_disk() { admin->__insert_or_replace_maintable_select_disktable(); } }; // --------------------------------------------------------------------------- @@ -116,6 +113,16 @@ static bool table_matches_current(SQLite3DB *db, const char *tbl, const char *de return db->check_table_structure(tbl, def) == 1; } +static bool create_table_in_schema(SQLite3DB* db, const char* schema, const char* definition) { + std::string query = definition; + const std::string prefix = "CREATE TABLE "; + if (query.compare(0, prefix.size(), prefix) != 0) { + return false; + } + query.insert(prefix.size(), std::string(schema) + "."); + return db->execute(query.c_str()); +} + // ============================================================================ // disk_upgrade_scheduler() tests // ============================================================================ @@ -579,81 +586,55 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { } static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { - TestDiskUpgrade t; - SQLite3DB *db = t.db(); - t.use_config_db_as_admin_db(); - - struct TableDefinition { - const char *name; - const char *definition; - }; - const TableDefinition tables[] = { - { "mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS }, - { "mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS }, - { "mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS }, - { "mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS }, - { "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS }, - { "mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS }, - { "mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES }, - { "mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS }, - { "mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES }, - { "mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING }, - { "mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS }, - { "mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS }, - { "mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES }, - { "mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS }, - { "global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES }, - { "scheduler", ADMIN_SQLITE_TABLE_SCHEDULER }, - { "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES }, - { "proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS }, - { "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS }, - { "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS }, - { "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES }, - { "pgsql_servers_ssl_params", ADMIN_SQLITE_TABLE_PGSQL_SERVERS_SSL_PARAMS }, - { "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES }, - { "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING }, - { "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS }, - { "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS }, - { "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES }, - { "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS } - }; - for (const TableDefinition& table : tables) { - db->execute(table.definition); - } - db->execute("ATTACH DATABASE ':memory:' AS disk"); - for (const TableDefinition& table : tables) { - std::string query = std::string("CREATE TABLE disk.") + table.name + - " AS SELECT * FROM main." + table.name + " WHERE 0"; - db->execute(query.c_str()); - } - - db->execute( + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + const bool schema_ready = db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && + db->execute("ATTACH DATABASE ':memory:' AS disk") && + create_table_in_schema(db, "disk", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(schema_ready, "Aurora disk round trip uses the current schemas in main and disk"); + + const bool rows_inserted = db->execute( "INSERT INTO mysql_aws_aurora_hostgroups (" "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" - ") VALUES (600,601,602,603,'.disk.example','disk round trip')" + ") VALUES " + "(600,601,602,603,'.disk.example','disk round trip')," + "(610,611,NULL,NULL,'.disk-null.example',NULL)" ); - // Model a node-local runtime field on the source. The disk projection must ignore it. - db->execute("ALTER TABLE main.mysql_aws_aurora_hostgroups ADD COLUMN bgd_status VARCHAR NOT NULL DEFAULT 'LOCAL'"); - db->execute("UPDATE main.mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS'"); - - t.save_mysql_servers_to_disk(); + ok(rows_inserted && copy_mysql_aws_aurora_hostgroups_to_disk(db), + "Aurora disk SAVE copies the configured projection"); ok(query_string(db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk SAVE preserves both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk SAVE preserves paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk SAVE preserves a NULL comment"); ok(query_int(db, "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, "Aurora disk SAVE excludes node-local bgd_status"); - db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); - t.load_mysql_servers_from_disk(); + ok(db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"), + "Aurora disk LOAD test clears the configured source table"); + ok(copy_mysql_aws_aurora_hostgroups_from_disk(db), + "Aurora disk LOAD copies the configured projection"); ok(query_string(db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk LOAD restores both configured green hostgroups"); - ok(query_string(db, - "SELECT bgd_status FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "LOCAL", - "Aurora disk LOAD leaves node-local bgd_status at its local default"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk LOAD restores paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk LOAD restores a NULL comment"); + delete db; } // ============================================================================ @@ -661,7 +642,7 @@ static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { // ============================================================================ int main() { - plan(71); + plan(78); test_init_minimal(); // scheduler tests diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index b1600a4215..84e1401608 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -10,10 +10,10 @@ #include "cpp.h" #include "MySQL_HostGroups_Manager.h" #include "ProxySQL_Admin_Tables_Definitions.h" +#include "proxysql_admin.h" #include #include -#include #include extern MySQL_HostGroups_Manager* MyHGM; @@ -46,34 +46,21 @@ class TestAuroraBGDRuntime { } static SQLite3DB* materialize_aurora_table(bool runtime) { - void* memory = calloc(1, sizeof(ProxySQL_Admin)); - ProxySQL_Admin* admin = reinterpret_cast(memory); SQLite3DB* db = new SQLite3DB(); db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - admin->admindb = db; - - if (runtime) { - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); - } else { - db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + const bool created = db->execute(runtime + ? ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS + : ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + SQLite3_result* resultset = MyHGM->dump_table_mysql(runtime + ? "runtime_mysql_aws_aurora_hostgroups" + : "mysql_aws_aurora_hostgroups"); + const bool materialized = created && + materialize_mysql_aws_aurora_hostgroups(db, resultset, runtime); + delete resultset; + if (!materialized) { + delete db; + return nullptr; } - - admin->save_mysql_servers_runtime_to_database(runtime); - free(admin); return db; } }; @@ -270,17 +257,16 @@ static void test_inactive_cross_row_validation() { delete candidate; } -static void test_legacy_projection_normalization() { +static void test_noncanonical_projection_rejected() { SQLite3_result* candidate = make_candidate(true); add_legacy_candidate_row(candidate, 190, 200); std::vector errors; SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); - ok(filtered->columns == 17 && filtered->rows_count == 1, - "legacy Aurora projection is normalized to the configured BGD shape"); - ok(filtered->rows[0]->fields[2] == nullptr && filtered->rows[0]->fields[3] == nullptr, - "legacy projection normalizes both green hostgroups to SQL NULL"); - ok(errors.empty(), "legacy Aurora projection remains valid"); + ok(filtered->columns == 17 && filtered->rows_count == 0, + "noncanonical Aurora projection publishes no rows"); + ok(contains_error(errors, "candidate projection", "green_writer_hostgroup"), + "noncanonical Aurora projection reports the first mismatched column"); delete filtered; delete candidate; } @@ -307,7 +293,7 @@ static void test_invalid_replacement_removes_previous_row() { delete initial; } -static void test_runtime_ownership_and_status() { +static void test_runtime_ownership() { SQLite3_result* initial = make_candidate(); add_candidate_row(initial, 300, 310, "301", "311", true); add_candidate_row(initial, 320, 330, nullptr, nullptr, true); @@ -325,55 +311,61 @@ static void test_runtime_ownership_and_status() { "SELECT GROUP_CONCAT(bgd_status, ',') FROM " "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); - const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); - SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); - ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); - delete dump; + SQLite3_result* configured_dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); + ok(configured_dump && configured_dump->columns == 17, + "Aurora configured dump excludes bgd_status"); + delete configured_dump; + SQLite3_result* runtime_dump = MyHGM->dump_table_mysql("runtime_mysql_aws_aurora_hostgroups"); + ok(runtime_dump && runtime_dump->columns == 18, + "Aurora runtime dump includes configured fields and bgd_status"); + delete runtime_dump; +} + +static void test_status_and_materialization() { + const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); ok(MyHGM->gen_global_mysql_servers_v2_checksum(0) == checksum_before_status, "node-local Aurora BGD status is excluded from the cluster checksum"); - const char* accepted_statuses[] = { - "NONE", - "AVAILABLE", - "SWITCHOVER_INITIATED", - "SWITCHOVER_IN_PROGRESS", - "SWITCHOVER_IN_POST_PROCESSING", - "SWITCHOVER_COMPLETED" - }; - bool accepted_all_statuses = true; - for (const char* status : accepted_statuses) { - MyHGM->update_aws_aurora_bgd_status(300, status); - accepted_all_statuses = accepted_all_statuses && hgm_query_string( - "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == status; - } - ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); - MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + SQLite3DB* configured_db = TestAuroraBGDRuntime::materialize_aurora_table(false); - ok(query_string(configured_db, - "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " - "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", - "SAVE from runtime preserves both configured green hostgroups"); - ok(query_int(configured_db, - "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, - "SAVE from runtime excludes bgd_status from configuration"); + ok(configured_db != nullptr, "configured Aurora table materializes without Admin object emulation"); + if (configured_db) { + ok(query_string(configured_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "SAVE from runtime preserves both configured green hostgroups"); + ok(query_int(configured_db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "SAVE from runtime excludes bgd_status from configuration"); + } else { + skip(2, "configured Aurora materialization failed"); + } delete configured_db; SQLite3DB* runtime_db = TestAuroraBGDRuntime::materialize_aurora_table(true); - ok(query_string(runtime_db, - "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " - "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", - "runtime materialization preserves both configured green hostgroups"); - ok(query_string(runtime_db, - "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", - "runtime materialization includes the node-local bgd_status"); + ok(runtime_db != nullptr, "runtime Aurora table materializes without Admin object emulation"); + if (runtime_db) { + ok(query_string(runtime_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "runtime materialization preserves both configured green hostgroups"); + ok(query_string(runtime_db, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", + "runtime materialization includes the node-local bgd_status"); + } else { + skip(2, "runtime Aurora materialization failed"); + } delete runtime_db; +} +static void test_reload_and_status_ordering() { + int green_writer = 0; + int green_reader = 0; SQLite3_result* unrelated_reload = make_candidate(); add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); TestAuroraBGDRuntime::reload(MyHGM, unrelated_reload); @@ -384,28 +376,21 @@ static void test_runtime_ownership_and_status() { green_writer == 302 && green_reader == 312, "configuration reload updates runtime green hostgroups"); - MyHGM->update_aws_aurora_bgd_status(300, "NOT_A_BGD_STATUS"); + SQLite3_result* status_after_reload = make_candidate(); + add_candidate_row(status_after_reload, 300, 310, "303", "313", true, "status after reload"); + TestAuroraBGDRuntime::reload(MyHGM, status_after_reload); + MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == "AVAILABLE", "invalid Aurora BGD status leaves runtime state unchanged"); - - SQLite3_result* concurrent_reload = make_candidate(); - add_candidate_row(concurrent_reload, 300, 310, "303", "313", true, "concurrent"); - std::thread reload_thread([concurrent_reload]() { - TestAuroraBGDRuntime::reload(MyHGM, concurrent_reload); - }); - std::thread status_thread([]() { - MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); - }); - reload_thread.join(); - status_thread.join(); - ok(hgm_query_string( - "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == "SWITCHOVER_IN_PROGRESS", "reload and status publication serialize without losing state"); + ) == "SWITCHOVER_IN_PROGRESS", "status publication after reload updates the reloaded row"); ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && green_writer == 303 && green_reader == 313, - "serialized reload publishes its configured values"); + "status publication after reload keeps its configured values"); +} +static void test_runtime_removal() { + int green_writer = 0; + int green_reader = 0; SQLite3_result* empty_reload = make_candidate(); TestAuroraBGDRuntime::reload(MyHGM, empty_reload); ok(hgm_query_int( @@ -416,17 +401,22 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(49); + plan(50); test_init_minimal(); test_schema_contract(); // 12 test_row_validation(); // 8 test_inactive_cross_row_validation(); // 4 - test_legacy_projection_normalization(); // 3 + test_noncanonical_projection_rejected(); // 2 test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 18 + ok(test_init_monitor() == 0, "test_init_monitor() succeeds"); // 1 + test_runtime_ownership(); // 4 + test_status_and_materialization(); // 8 + test_reload_and_status_ordering(); // 4 + test_runtime_removal(); // 2 + test_cleanup_monitor(); test_cleanup_hostgroups(); test_cleanup_minimal(); From f98563eb0a77c831ca90f39066ac4aaac2fe64c3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 22:50:40 +0000 Subject: [PATCH 16/26] refactor: share AWS BGD simulator topology --- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 34 +-- include/SQLite3_Server.h | 14 +- src/SQLite3_Server.cpp | 84 +++--- test/tap/tap/Makefile | 9 +- test/tap/tap/bgd_simulator.cpp | 223 ++++++++++++++++ test/tap/tap/bgd_simulator.h | 60 +++++ test/tap/tap/rds_bgd_simulator.cpp | 220 +--------------- test/tap/tap/rds_bgd_simulator.h | 244 +----------------- test/tap/tap/rds_bgd_tap.h | 40 +-- .../test_rds_bgd_automatic_discovery-t.cpp | 22 +- .../test_rds_bgd_concurrent_isolation-t.cpp | 42 +-- ...nfig_refresh_after_writer_completion-t.cpp | 28 +- ...st_rds_bgd_configuration_persistence-t.cpp | 22 +- ...st_rds_bgd_disable_during_switchover-t.cpp | 14 +- .../tests/test_rds_bgd_explicit_startup-t.cpp | 18 +- ...st_rds_bgd_green_membership_ordering-t.cpp | 22 +- .../test_rds_bgd_green_pool_cleanup-t.cpp | 28 +- .../test_rds_bgd_late_entry_completed-t.cpp | 22 +- ...est_rds_bgd_late_entry_writer_phases-t.cpp | 26 +- test/tap/tests/test_rds_bgd_probe_tls-t.cpp | 38 +-- .../tests/test_rds_bgd_reader_policy-t.cpp | 24 +- ...st_rds_bgd_reader_switchover_cleanup-t.cpp | 36 +-- ...est_rds_bgd_remove_during_switchover-t.cpp | 14 +- .../test_rds_bgd_repeated_deployment-t.cpp | 38 +-- test/tap/tests/test_rds_bgd_rollback-t.cpp | 24 +- test/tap/tests/test_rds_bgd_smoke-t.cpp | 14 +- .../test_rds_bgd_topology_empty_absent-t.cpp | 44 ++-- .../tests/test_rds_bgd_topology_errors-t.cpp | 50 ++-- .../test_rds_bgd_worker_config_refresh-t.cpp | 32 +-- ...est_rds_bgd_worker_hostgroup_refresh-t.cpp | 22 +- .../test_rds_bgd_writer_switchover-t.cpp | 34 +-- 31 files changed, 694 insertions(+), 848 deletions(-) create mode 100644 test/tap/tap/bgd_simulator.cpp create mode 100644 test/tap/tap/bgd_simulator.h diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 7c228a93bc..01777d12d8 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -78,7 +78,7 @@ Create the following tables in `SQLite3_Server::init()` and store them in the existing persistent `GloVars.sqlite3serverdb` database: ```sql -CREATE TABLE RDS_BGD_CONTROL ( +CREATE TABLE AWS_BGD_CONTROL ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), @@ -87,7 +87,7 @@ CREATE TABLE RDS_BGD_CONTROL ( PRIMARY KEY (backend_ip, backend_port) ); -CREATE TABLE RDS_BGD_TOPOLOGY ( +CREATE TABLE AWS_BGD_TOPOLOGY ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, @@ -99,7 +99,7 @@ CREATE TABLE RDS_BGD_TOPOLOGY ( PRIMARY KEY (backend_ip, backend_port, row_order) ); -CREATE TABLE RDS_BGD_PROBE_LOG ( +CREATE TABLE AWS_BGD_PROBE_LOG ( sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, @@ -141,7 +141,7 @@ The TAP helper publishes control and topology changes atomically. Monitor probes read committed simulator state without holding a cross-query snapshot. The supported states are: -| `RDS_BGD_CONTROL` state | Topology rows | Meaning | +| `AWS_BGD_CONTROL` state | Topology rows | Meaning | |---|---|---| | No backend row | None | Backend is unconfigured; topology is absent. | | `topology_present=1`, `error_code=0` | One or more | Return the configured topology. | @@ -180,7 +180,7 @@ configured simulator errors may use `HY000` unless a test requires a specific mapping. Every handled topology-table check or metadata query appends one row to -`RDS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` +`AWS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` preserves order, `probe_kind` identifies the query, `backend_ip, backend_port` identify the destination, and `encrypted` records the accepted stream's TLS state. @@ -205,7 +205,7 @@ returns one `read_only` column. Table writes refresh the cache, and a missing entry returns the safe default `read_only=1`. BGD topology tasks send the production topology queries unchanged. Read-only -handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. +handling does not consult `AWS_BGD_CONTROL` or write `AWS_BGD_PROBE_LOG`. ## TAP Helper API @@ -246,7 +246,7 @@ API; the helper closes it when destroyed. `read_only_update()` changes the ### Topology and Host Types ```cpp -struct RDS_BGD_Topology_Row { +struct BGD_Topology_Row { std::string id; std::string endpoint; int port; @@ -264,7 +264,7 @@ struct RDS_BGD_Host { }; ``` -`RDS_BGD_Topology_Row` represents one `mysql.rds_topology` row using +`BGD_Topology_Row` represents one `mysql.rds_topology` row using C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing hostname and simulator-facing IP together. @@ -282,7 +282,7 @@ public: std::vector get_blue_endpoints(); std::vector get_green_endpoints(); std::vector get_endpoints(); - std::vector get_topology(std::string status); + std::vector get_topology(std::string status); }; ``` @@ -300,7 +300,7 @@ reader mapping. ```cpp int topology_update( std::vector backends, - std::vector rows); + std::vector rows); int topology_delete(std::vector backends); @@ -330,26 +330,26 @@ not inherit simulator state from an earlier binary. ### Probe-Log Operations ```cpp -enum class RDS_BGD_Probe_Kind { +enum class BGD_Probe_Kind { table_check, metadata, }; -struct RDS_BGD_Probe_Log { +struct BGD_Probe_Log { uint64_t sequence_id; Endpoint backend; - RDS_BGD_Probe_Kind probe_kind; + BGD_Probe_Kind probe_kind; bool encrypted; }; rc_t probe_log_last_sequence(); -rc_t> probe_log_since(uint64_t sequence_id); +rc_t> probe_log_since(uint64_t sequence_id); -rc_t wait_for_probe_log( +rc_t wait_for_probe_log( uint64_t sequence_id, Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, + BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted = -1); ``` @@ -366,7 +366,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator simulator {}; + BGD_Simulator simulator {}; if (setup(cl, admin, simulator) != EXIT_SUCCESS) return exit_status(); diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index a2ceaabe2a..20b5f30ab5 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -51,13 +51,13 @@ class SQLite3_Server { std::unordered_map grouprep_map; std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; std::unordered_map readonly_map; -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - std::vector *tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + std::vector *tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG std::unordered_map> replicationlag_map; std::vector* tables_defs_replicationlag; @@ -97,14 +97,14 @@ class SQLite3_Server { void init_grouprep_ifaces_string(std::string& s); group_rep_status grouprep_test_value(const std::string& srv_addr); #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) pthread_mutex_t test_readonly_mutex; void load_readonly_table(MySQL_Session *sess); int readonly_test_value(char *p); int readonly_map_size() { return readonly_map.size(); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG pthread_mutex_t test_replicationlag_mutex; void load_replicationlag_table(MySQL_Session* sess); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index d2e7d3414f..093b284045 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -11,7 +11,7 @@ #include "proxysql_utils.h" #include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" -#ifdef TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) #include "MySQL_Monitor.hpp" #endif @@ -285,7 +285,7 @@ class sqlite3server_main_loop_listeners { static sqlite3server_main_loop_listeners S_amll; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) static void ensure_readonly_table(SQLite3_Server *server, MySQL_Session *sess) { if (server->readonly_map_size() == 0) { server->load_readonly_table(sess); @@ -854,17 +854,17 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (run_query) { #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)==0) { -#ifdef TEST_RDS_BGD - const bool rds_bgd_table_check = +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + const bool aws_bgd_table_check = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK) == 0; - const bool rds_bgd_metadata = + const bool aws_bgd_metadata = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY) == 0; - if (rds_bgd_table_check || rds_bgd_metadata) { + if (aws_bgd_table_check || aws_bgd_metadata) { if (sess->client_myds->proxy_addr.addr == NULL || sess->client_myds->proxy_addr.port <= 0) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator could not identify the accepted backend address"); + "AWS BGD simulator could not identify the accepted backend address"); run_query=false; } else { SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; @@ -875,16 +875,16 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p std::to_string(backend_port) }; const std::string log_query { - "INSERT INTO RDS_BGD_PROBE_LOG" + "INSERT INTO AWS_BGD_PROBE_LOG" "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + backend_ip + "'," + std::to_string(backend_port) + ",'" + - (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (aws_bgd_table_check ? "table_check" : "metadata") + "'," + (sess->client_myds->encrypted ? "1" : "0") + ")" }; if (!sqlite_sess->sessdb->execute(log_query.c_str())) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator failed to record the topology probe"); + "AWS BGD simulator failed to record the topology probe"); run_query=false; } else { char *control_error=NULL; @@ -892,7 +892,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int control_affected_rows=0; SQLite3_result *control_result=NULL; const std::string control_query { - "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + "SELECT topology_present,error_code,error_msg FROM AWS_BGD_CONTROL WHERE " + predicate }; sqlite_sess->sessdb->execute_statement( @@ -917,7 +917,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } delete control_result; - if (run_query && rds_bgd_table_check) { + if (run_query && aws_bgd_table_check) { const char* topology_sql = topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"; static constexpr size_t topology_sql_len = sizeof("SELECT 1") - 1; @@ -940,7 +940,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } else if (run_query) { const std::string topology_query { "SELECT id,endpoint,topology_port AS port,role,status " - "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + + "FROM AWS_BGD_TOPOLOGY WHERE " + predicate + " ORDER BY row_order" }; l_free(query_length,query); @@ -951,7 +951,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } -#endif // TEST_RDS_BGD +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); @@ -1034,7 +1034,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, k_select_read_only_len)==0 && query_no_space_length > k_select_read_only_len+5) { pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); @@ -1048,7 +1048,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p query_length = formatted_query.size() + 1; pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG const bool replica_status = strncasecmp("SELECT REPLICA STATUS ", query_no_space, k_select_replica_status_len) == 0; const uint64_t addr_offset { @@ -1150,7 +1150,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p bool deprecate_eof = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans, deprecate_eof); delete resultset; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)) { if (strstr(query_no_space,(char *)"READONLY_STATUS")) { // the table is writable @@ -1159,7 +1159,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if (strncasecmp("SELECT", query_no_space, 6)) { if (strstr(query_no_space, (char*)"REPLICATIONLAG_HOST_STATUS")) { @@ -1469,15 +1469,15 @@ SQLite3_Server::~SQLite3_Server() { delete tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) drop_tables_defs(tables_defs_readonly); delete tables_defs_readonly; #endif -#ifdef TEST_RDS_BGD - drop_tables_defs(tables_defs_rds_bgd); - delete tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + drop_tables_defs(tables_defs_aws_bgd); + delete tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD }; #ifdef TEST_AURORA @@ -1579,13 +1579,13 @@ SQLite3_Server::SQLite3_Server() { #ifdef TEST_GROUPREP init_grouprep_ifaces_string(s); #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) // Read-only simulation listens on all IPs because it can simulate many clusters. if (!s.empty()) s += ";"; s += "0.0.0.0:3306"; pthread_mutex_init(&test_readonly_mutex, NULL); -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG // for replication test we listen on all IPs if (!s.empty()) @@ -2041,7 +2041,7 @@ bool SQLite3_Server::init() { check_and_build_standard_tables(sessdb, tables_defs_grouprep); GloAdmin->enable_grouprep_testing(); #endif // TEST_GALERA -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) tables_defs_readonly = new std::vector; insert_into_tables_defs(tables_defs_readonly, (const char *)"READONLY_STATUS", @@ -2050,32 +2050,32 @@ bool SQLite3_Server::init() { #ifdef TEST_READONLY GloAdmin->enable_readonly_testing(); #endif // TEST_READONLY -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - tables_defs_rds_bgd = new std::vector; - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_CONTROL", - (const char *)"CREATE TABLE RDS_BGD_CONTROL (" +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + tables_defs_aws_bgd = new std::vector; + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_CONTROL", + (const char *)"CREATE TABLE AWS_BGD_CONTROL (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, " "topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), " "error_code INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', " "PRIMARY KEY (backend_ip, backend_port))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_TOPOLOGY", - (const char *)"CREATE TABLE RDS_BGD_TOPOLOGY (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_TOPOLOGY", + (const char *)"CREATE TABLE AWS_BGD_TOPOLOGY (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, " "id TEXT NOT NULL, endpoint TEXT NOT NULL, topology_port INTEGER NOT NULL, " "role TEXT NOT NULL, status TEXT NOT NULL, " "PRIMARY KEY (backend_ip, backend_port, row_order))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_PROBE_LOG", - (const char *)"CREATE TABLE RDS_BGD_PROBE_LOG (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_PROBE_LOG", + (const char *)"CREATE TABLE AWS_BGD_PROBE_LOG (" "sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, " "backend_port INTEGER NOT NULL, probe_kind TEXT NOT NULL " "CHECK (probe_kind IN ('table_check','metadata')), encrypted INTEGER NOT NULL " "CHECK (encrypted IN (0,1)))"); - check_and_build_standard_tables(sessdb, tables_defs_rds_bgd); -#endif // TEST_RDS_BGD + check_and_build_standard_tables(sessdb, tables_defs_aws_bgd); +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG tables_defs_replicationlag = new std::vector; insert_into_tables_defs(tables_defs_replicationlag, @@ -2205,7 +2205,7 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, myds->DSS=STATE_SLEEP; } -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired GloAdmin->mysql_servers_wrlock(); @@ -2246,7 +2246,7 @@ int SQLite3_Server::readonly_test_value(char *p) { } return rc; } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG void SQLite3_Server::load_replicationlag_table(MySQL_Session* sess) { diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index ead866e075..6e090f4ad9 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -83,11 +83,14 @@ mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h cluster_simulator.h utils.h +bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h + $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp new file mode 100644 index 0000000000..c0f476e706 --- /dev/null +++ b/test/tap/tap/bgd_simulator.cpp @@ -0,0 +1,223 @@ +#include "bgd_simulator.h" + +#include +#include +#include +#include + +#include "tap.h" + +using namespace std; + +namespace { + +const char* probe_kind_string(BGD_Probe_Kind kind) { + return kind == BGD_Probe_Kind::table_check ? "table_check" : "metadata"; +} + +rc_t parse_probe_kind(string value) { + if (value == "table_check") { + return { EXIT_SUCCESS, BGD_Probe_Kind::table_check }; + } + if (value == "metadata") { + return { EXIT_SUCCESS, BGD_Probe_Kind::metadata }; + } + return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; +} + +} // namespace + +int BGD_Simulator::topology_update(vector backends, vector rows) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + string predicate { backend_predicate(backend) }; + statements.push_back("DELETE FROM AWS_BGD_TOPOLOGY WHERE " + predicate); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + + for (size_t row_order = 0; row_order < rows.size(); ++row_order) { + BGD_Topology_Row& row = rows[row_order]; + statements.push_back( + "INSERT INTO AWS_BGD_TOPOLOGY" + "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + to_string(row.port) + "," + + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); + } + } + + return execute_transaction(statements); +} + +int BGD_Simulator::topology_delete(vector backends) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::topology_drop(vector backends) { + return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +} + +int BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + bool topology_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + if (!topology_present) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + } + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::cleanup() { + vector statements { + "DELETE FROM READONLY_STATUS", + "DELETE FROM AWS_BGD_TOPOLOGY", + "DELETE FROM AWS_BGD_CONTROL", + "DELETE FROM AWS_BGD_PROBE_LOG", + }; + return execute_transaction(statements); +} + +rc_t BGD_Simulator::probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + + auto [rc, rows] = mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_BGD_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::probe_log_since(uint64_t sequence_id) { + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " + "FROM AWS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 5) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, kind] = parse_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + kind, + atoi(row[4].c_str()) != 0, + }); + } + + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (BGD_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (BGD_Probe_Log& log : logs) { + diag( + "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", + backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); + return { ETIMEDOUT, {} }; +} + +int BGD_Simulator::execute_transaction(vector& statements) { + if (execute("START TRANSACTION") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (string& statement : statements) { + if (execute(statement) != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + } + if (execute("COMMIT") != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +string BGD_Simulator::backend_predicate(Endpoint backend) { + return "backend_ip=" + sql_quote(backend.host) + + " AND backend_port=" + to_string(backend.port); +} diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h new file mode 100644 index 0000000000..a98013319d --- /dev/null +++ b/test/tap/tap/bgd_simulator.h @@ -0,0 +1,60 @@ +#ifndef TAP_BGD_SIMULATOR_H +#define TAP_BGD_SIMULATOR_H + +#include +#include +#include + +#include "cluster_simulator.h" +#include "utils.h" + +using namespace std; + +/** One row returned by the simulated mysql.rds_topology table. */ +struct BGD_Topology_Row { + string id; + string endpoint; + int port; + string role; + string status; +}; + +/** Identifies a topology query recorded by the shared AWS BGD simulator. */ +enum class BGD_Probe_Kind { + table_check, + metadata, +}; + +/** One topology query observed by the shared AWS BGD simulator. */ +struct BGD_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +/** Controls shared AWS BGD topology responses and reads their probe log. */ +class BGD_Simulator : public Cluster_Simulator { +public: + int topology_update(vector backends, vector rows); + int topology_delete(vector backends); + int topology_drop(vector backends); + int topology_error(vector backends, int error_code, string error_msg); + int cleanup(); + + rc_t probe_log_last_sequence(); + rc_t> probe_log_since(uint64_t sequence_id); + rc_t wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1 + ); + +protected: + static string backend_predicate(Endpoint backend); + int execute_transaction(vector& statements); +}; + +#endif // TAP_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index 5b53f0aad4..673aa50790 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -1,28 +1,7 @@ #include "rds_bgd_simulator.h" -#include -#include -#include -#include - -#include "tap.h" - using namespace std; -const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { - return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; -} - -rc_t parse_probe_kind(string value) { - if (value == "table_check") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; - } - if (value == "metadata") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::metadata }; - } - return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; -} - Endpoint RDS_BGD_Host::endpoint() { return { ip, port }; } @@ -54,7 +33,7 @@ vector RDS_BGD_Cluster::get_endpoints() { return endpoints; } -vector RDS_BGD_Cluster::get_topology(string status) { +vector RDS_BGD_Cluster::get_topology(string status) { return { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, @@ -62,200 +41,3 @@ vector RDS_BGD_Cluster::get_topology(string status) { "BLUE_GREEN_DEPLOYMENT_TARGET", status }, }; } - -int RDS_BGD_Simulator::topology_update(vector backends, vector rows) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - string predicate { backend_predicate(backend) }; - statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - - for (size_t row_order = 0; row_order < rows.size(); ++row_order) { - RDS_BGD_Topology_Row& row = rows[row_order]; - statements.push_back( - "INSERT INTO RDS_BGD_TOPOLOGY" - "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - to_string(row_order) + "," + sql_quote(row.id) + "," + - sql_quote(row.endpoint) + "," + to_string(row.port) + "," + - sql_quote(row.role) + "," + sql_quote(row.status) + ")"); - } - } - - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_delete(vector backends) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_drop(vector backends) { - return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); -} - -int RDS_BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { - if (backends.empty() || error_code == 0) { - return EXIT_FAILURE; - } - - bool topology_present = error_code != 1146; - vector statements {}; - for (Endpoint& backend : backends) { - if (!topology_present) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - } - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + - sql_quote(error_msg) + ")"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::cleanup() { - vector statements { - "DELETE FROM READONLY_STATUS", - "DELETE FROM RDS_BGD_TOPOLOGY", - "DELETE FROM RDS_BGD_CONTROL", - "DELETE FROM RDS_BGD_PROBE_LOG", - }; - return execute_transaction(statements); -} - -rc_t RDS_BGD_Simulator::probe_log_last_sequence() { - if (connection() == nullptr) { - return { EXIT_FAILURE, 0 }; - } - - auto [rc, rows] = mysql_query_ext_rows( - connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG"); - if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { - return { EXIT_FAILURE, 0 }; - } - - return { - EXIT_SUCCESS, - static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) - }; -} - -rc_t> RDS_BGD_Simulator::probe_log_since( - uint64_t sequence_id) -{ - if (connection() == nullptr) { - return { EXIT_FAILURE, {} }; - } - - string query { - "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " - "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + - " ORDER BY sequence_id" - }; - auto [rc, rows] = mysql_query_ext_rows(connection(), query); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - - vector logs {}; - for (mysql_res_row& row : rows) { - if (row.size() != 5) { - return { EXIT_FAILURE, {} }; - } - auto [kind_rc, kind] = parse_probe_kind(row[3]); - if (kind_rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - logs.push_back({ - static_cast(strtoull(row[0].c_str(), nullptr, 10)), - { row[1], atoi(row[2].c_str()) }, - kind, - atoi(row[4].c_str()) != 0, - }); - } - - return { EXIT_SUCCESS, move(logs) }; -} - -rc_t RDS_BGD_Simulator::wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted) -{ - uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; - do { - auto [rc, logs] = probe_log_since(sequence_id); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - for (RDS_BGD_Probe_Log& log : logs) { - if (log.backend.host == backend.host && log.backend.port == backend.port && - log.probe_kind == probe_kind && - (encrypted < 0 || log.encrypted == (encrypted != 0))) { - return { EXIT_SUCCESS, log }; - } - } - usleep(50000); - } while (monotonic_time() < deadline); - - auto [rc, logs] = probe_log_since(sequence_id); - if (rc == EXIT_SUCCESS) { - for (RDS_BGD_Probe_Log& log : logs) { - diag( - "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", - static_cast(log.sequence_id), - log.backend.host.c_str(), log.backend.port, - probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); - } - } - diag( - "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", - backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); - return { ETIMEDOUT, {} }; -} - -int RDS_BGD_Simulator::execute_transaction(vector& statements) { - if (execute("START TRANSACTION") != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - for (string& statement : statements) { - if (execute(statement) != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - } - if (execute("COMMIT") != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -string RDS_BGD_Simulator::backend_predicate(Endpoint backend) { - return "backend_ip=" + sql_quote(backend.host) + - " AND backend_port=" + to_string(backend.port); -} diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h index 2827d39435..a112d760a3 100644 --- a/test/tap/tap/rds_bgd_simulator.h +++ b/test/tap/tap/rds_bgd_simulator.h @@ -1,258 +1,36 @@ #ifndef TAP_RDS_BGD_SIMULATOR_H #define TAP_RDS_BGD_SIMULATOR_H -#include #include #include -#include "cluster_simulator.h" -#include "utils.h" +#include "bgd_simulator.h" using namespace std; -/** - * @brief Represents one row returned by the simulated `mysql.rds_topology` table. - */ -struct RDS_BGD_Topology_Row { - string id; ///< RDS topology node identifier. - string endpoint; ///< RDS hostname exposed by the topology row. - int port; ///< MySQL port exposed by the topology row. - string role; ///< Blue/green deployment role reported by RDS. - string status; ///< Blue/green deployment status reported by RDS. -}; - -/** - * @brief Describes one RDS BGD host and its fixed simulator address. - */ +/** Describes one RDS BGD host and its fixed simulator address. */ struct RDS_BGD_Host { - string hostname; ///< AWS-style RDS hostname configured in ProxySQL. - string ip; ///< Fixed loopback address used by the simulator. - int port; ///< MySQL listener port shared by the hostname and IP. + string hostname; + string ip; + int port; - /** - * @brief Returns the IP/port endpoint used for topology simulation. - * - * @return Simulator endpoint containing this host's IP address and port. - */ Endpoint endpoint(); - - /** - * @brief Returns the hostname/port endpoint used for read-only simulation. - * - * @return Simulator endpoint containing this host's RDS hostname and port. - */ Endpoint host_endpoint(); }; -/** - * @brief Holds the blue and green hosts participating in one simulated RDS BGD cluster. - * - * @details TAP tests populate the cluster with the deployment topology required by each - * scenario. Helper methods derive writer endpoints and AWS topology rows from - * the configured hosts. - */ +/** Holds the blue and green hosts in one simulated RDS BGD deployment. */ class RDS_BGD_Cluster { public: - RDS_BGD_Host blue_writer; ///< Source writer configured in ProxySQL. - RDS_BGD_Host green_writer; ///< Target writer discovered from the topology. - vector blue_readers; ///< Source readers configured in ProxySQL. - vector green_readers; ///< Target readers discovered from the topology. + RDS_BGD_Host blue_writer; + RDS_BGD_Host green_writer; + vector blue_readers; + vector green_readers; - /** - * @brief Returns both writer IP/port endpoints for topology simulation. - * - * @return Blue and green writer endpoints keyed by simulator IP address. - */ vector get_writers(); - - /** - * @brief Returns the blue writer and configured blue readers. - * - * @return Blue deployment endpoints keyed by simulator IP address. - */ vector get_blue_endpoints(); - - /** - * @brief Returns the green writer and configured green readers. - * - * @return Green deployment endpoints keyed by simulator IP address. - */ vector get_green_endpoints(); - - /** - * @brief Returns every simulator IP/port endpoint in this cluster. - * - * @details Includes both writers and all configured blue and green readers. - * Tests use this list when resetting or publishing topology for a complete - * simulated deployment. - * - * @return Writer and reader endpoints keyed by simulator IP address. - */ vector get_endpoints(); - - /** - * @brief Builds the topology rows published by the simulated writers. - * - * @details Creates one source row for the blue writer and one target row for the green - * writer. The supplied deployment status is applied to both rows. - * - * @param status RDS blue/green deployment status to publish. - * - * @return Source and target rows for the simulated topology table. - */ - vector get_topology(string status); -}; - -/** - * @brief Identifies the RDS BGD monitor query recorded in the simulator probe log. - */ -enum class RDS_BGD_Probe_Kind { - table_check, ///< Query checking whether `mysql.rds_topology` exists. - metadata, ///< Query fetching rows from `mysql.rds_topology`. -}; - -/** - * @brief Describes one RDS BGD monitor query observed by the simulator. - */ -struct RDS_BGD_Probe_Log { - uint64_t sequence_id; ///< Monotonically increasing probe-log sequence. - Endpoint backend; ///< Accepted backend IP address and port. - RDS_BGD_Probe_Kind probe_kind; ///< Type of topology query observed. - bool encrypted; ///< Whether the monitor connection used TLS. -}; - -/** - * @brief Controls RDS BGD topology responses and inspects monitor probes from TAP tests. - * - * @details Publishes per-backend topology rows, empty results, missing tables, or MySQL - * errors. It also reads the ordered probe log generated by the SQLite3-server - * simulator and reuses `Cluster_Simulator` for shared control operations. - */ -class RDS_BGD_Simulator : public Cluster_Simulator { -public: - /** - * @brief Replaces the simulated topology returned by each backend. - * - * @details Deletes existing topology rows before inserting the supplied rows. The - * topology table is marked present, configured errors are cleared, and the - * complete update is applied atomically across all supplied backends. - * - * @param backends Backend IP/port endpoints that must return the topology. - * @param rows Topology rows to publish on each backend. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_update(vector backends, vector rows); - - /** - * @brief Configures each backend to return an empty topology result. - * - * @details Deletes all topology rows associated with the supplied backends while keeping - * the topology table present and clearing any configured metadata error. - * - * @param backends Backend IP/port endpoints that must return an empty result. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_delete(vector backends); - - /** - * @brief Configures each backend to report that the topology table does not exist. - * - * @param backends Backend IP/port endpoints that must return MySQL error 1146. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_drop(vector backends); - - /** - * @brief Configures a MySQL error for topology queries on each backend. - * - * @details Stores the nonzero error code and message returned by subsequent metadata - * probes. Error 1146 marks the topology table absent and removes its existing - * rows; other error codes leave the table marked present. - * - * @param backends Backend IP/port endpoints that must return the error. - * @param error_code Nonzero MySQL error code to return. - * @param error_msg MySQL error message to return. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_error(vector backends, int error_code, string error_msg); - - /** - * @brief Removes all read-only, topology-control, topology-row, and probe state. - * - * @return EXIT_SUCCESS when the simulator state is empty; EXIT_FAILURE otherwise. - */ - int cleanup(); - - /** - * @brief Reads the latest sequence from the RDS BGD probe log. - * - * @return EXIT_SUCCESS and the latest sequence, or zero when the log is empty; - * EXIT_FAILURE and zero when the query fails. - */ - rc_t probe_log_last_sequence(); - - /** - * @brief Returns probe-log records newer than a sequence. - * - * @details Selects records with `sequence_id` strictly greater than the supplied value - * and preserves database sequence order in the returned vector. - * - * @param sequence_id Last probe-log sequence already observed by the TAP test. - * - * @return EXIT_SUCCESS and the matching records; EXIT_FAILURE and an empty vector - * when the query or record parsing fails. - */ - rc_t> probe_log_since(uint64_t sequence_id); - - /** - * @brief Waits for a matching RDS BGD probe-log record. - * - * @details Matches records newer than `sequence_id` by backend and probe kind. TLS state - * is matched when `encrypted` is zero or one; `-1` accepts either state. Observed - * probes are emitted through TAP diagnostics when the wait expires. - * - * @param sequence_id Last probe-log sequence observed before the expected probe. - * @param backend Backend IP/port endpoint expected to receive the probe. - * @param probe_kind Type of topology query expected. - * @param timeout_ms Maximum time to wait in milliseconds. - * @param encrypted Expected TLS state, or -1 to accept either state. - * - * @return EXIT_SUCCESS and the matching record; ETIMEDOUT and an empty record when - * the deadline expires; EXIT_FAILURE and an empty record when log retrieval fails. - */ - rc_t wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted = -1 - ); - -private: - /** - * @brief Builds the SQL predicate identifying one simulated backend. - * - * @param backend Backend IP/port endpoint to match. - * - * @return SQL predicate matching the backend control-table key. - */ - static string backend_predicate(Endpoint backend); - - /** - * @brief Executes simulator control statements in one transaction. - * - * @details Executes the supplied statements in order and commits only after every - * statement succeeds. A statement or commit failure triggers a rollback. - * - * @param statements SQL statements to execute atomically. - * - * @return EXIT_SUCCESS when the transaction commits; EXIT_FAILURE otherwise. - */ - int execute_transaction(vector& statements); + vector get_topology(string status); }; #endif // TAP_RDS_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h index 619450c7af..16bb5091f7 100644 --- a/test/tap/tap/rds_bgd_tap.h +++ b/test/tap/tap/rds_bgd_tap.h @@ -90,8 +90,8 @@ struct BGD_Hostgroups { int green_reader; }; -inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }); } @@ -101,7 +101,7 @@ inline vector bgd_topology_with_readers(RDS_BGD_Cluster& c return rows; } -inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +inline int bgd_set_writer_read_only_0(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (sim.read_only_update(cluster.blue_writer.host_endpoint(), false) != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated blue writer"); return EXIT_FAILURE; @@ -114,12 +114,12 @@ inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& c return EXIT_SUCCESS; } -inline int bgd_set_host_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_0(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), false); return rc; } -inline int bgd_set_host_read_only_1(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_1(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), true); return rc; } @@ -310,7 +310,7 @@ inline rc_t bgd_backend_ip_echo(MYSQL* proxy) { } inline rc_t bgd_probe_count_since( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, RDS_BGD_Probe_Kind kind) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, BGD_Probe_Kind kind) { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { @@ -319,7 +319,7 @@ inline rc_t bgd_probe_count_since( } uint64_t count = 0; - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { bool backend_matches = log.backend.host == backend.host && log.backend.port == backend.port; @@ -417,19 +417,19 @@ inline int bgd_wait_for_server_placement( return rc; } -inline rc_t bgd_wait_for_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, - RDS_BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) +inline rc_t bgd_wait_for_probe_from_backends( + BGD_Simulator& sim, uint64_t sequence, vector backends, + BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) { uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; do { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { - rc_t result { EXIT_FAILURE, {} }; + rc_t result { EXIT_FAILURE, {} }; return result; } - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { for (const Endpoint& backend : backends) { bool backend_matches = log.backend.host == backend.host && @@ -439,7 +439,7 @@ inline rc_t bgd_wait_for_probe_from_backends( encrypted < 0 || log.encrypted == (encrypted != 0); if (backend_matches && kind_matches && encryption_matches) { - rc_t result { EXIT_SUCCESS, log }; + rc_t result { EXIT_SUCCESS, log }; return result; } } @@ -448,7 +448,7 @@ inline rc_t bgd_wait_for_probe_from_backends( usleep(50000); } while (monotonic_time() < deadline); - rc_t result { ETIMEDOUT, {} }; + rc_t result { ETIMEDOUT, {} }; return result; } @@ -459,10 +459,10 @@ inline rc_t bgd_wait_for_probe_from_backends( * after the given sequence. */ inline int bgd_expect_no_table_check( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::table_check, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::table_check, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -478,12 +478,12 @@ inline int bgd_expect_no_table_check( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) { vector backends { backend }; auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -499,10 +499,10 @@ inline int bgd_expect_no_metadata_probe( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp index f7ac46e125..b9ed27422c 100644 --- a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -36,7 +36,7 @@ struct TestState { uint64_t absent_available_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -141,7 +141,7 @@ bool runtime_bgd_row_count_matches(MYSQL* admin, int writer_hostgroup, int expec * - Verify one auto-generated runtime row with NULL green hostgroups. * - Verify automatic discovery does not create a persistent BGD row. */ -int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_before_writer(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.topology_first; BGD_Hostgroups& hg = state.topology_first_hg; @@ -151,7 +151,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish topology-first AVAILABLE topology"); @@ -189,7 +189,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no runtime or persistent BGD row is created. * - Publish AVAILABLE topology and verify automatic row creation. */ -int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_absent_then_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; @@ -220,7 +220,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [absent_probe_rc, absent_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (absent_probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not issue the absent-table metadata probe"); return EXIT_FAILURE; @@ -236,7 +236,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.absent_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 820"); @@ -250,7 +250,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: wHG 820 did not probe the AVAILABLE green writer"); return EXIT_FAILURE; @@ -269,12 +269,12 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify runtime contains one auto-generated BGD row. * - Verify the automatic row remains absent from persistent configuration. */ -int test_repeated_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_repeated_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; auto [probe_rc, probe] = sim.wait_for_probe_log( - state.absent_available_sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + state.absent_available_sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic wHG 820 did not continue green-writer metadata polling"); return EXIT_FAILURE; @@ -291,7 +291,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp index 14329cd04f..2b5fff8841 100644 --- a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp @@ -36,7 +36,7 @@ struct TestState { BGD_Hostgroups cluster_3_hg { 1430, 1431, 1432, 1433 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,7 +95,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -116,13 +116,13 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_available( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) { if (configure_read_only_values(sim, cluster) != EXIT_SUCCESS) { return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); if (sim.topology_update(cluster.get_endpoints(), topology) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -215,7 +215,7 @@ int replace_cluster_1_green_membership(MYSQL* admin, TestState& state) { * - Use plaintext green metadata for clusters 1 and 3 and TLS for cluster 2. * - Verify that each BGD row reaches AVAILABLE through its own green writer. */ -int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_three_workers_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { auto [seq_rc, seq] = sim.probe_log_last_sequence(); if (seq_rc != EXIT_SUCCESS) { diag("Error: failed to read the probe sequence before starting three BGD workers"); @@ -241,7 +241,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_1.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1410 did not probe its plaintext green writer"); @@ -250,7 +250,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1410 reports AVAILABLE from its own plaintext green writer"); auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( - seq, state.cluster_2.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_2.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not probe its TLS green writer"); @@ -259,7 +259,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1420 reports AVAILABLE from its own TLS green writer"); auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( - seq, state.cluster_3.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_3.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not probe its plaintext green writer"); @@ -278,8 +278,8 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - After each change, verify that the other two statuses and blue-writer * placements remain unchanged. */ -int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { - vector cluster_1_topology = +int test_independent_phase_changes(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + vector cluster_1_topology = topology_with_reader_pair(state.cluster_1, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1.get_endpoints(), cluster_1_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS for BGD wHG 1410"); @@ -308,7 +308,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_2_available && cluster_3_available, "advancing wHG 1410 leaves wHG 1420 and wHG 1430 in AVAILABLE with unchanged blue placement"); - vector cluster_2_topology = + vector cluster_2_topology = topology_with_reader_pair(state.cluster_2, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(state.cluster_2.get_endpoints(), cluster_2_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING for BGD wHG 1420"); @@ -338,7 +338,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_1_in_progress && cluster_3_still_available, "advancing wHG 1420 preserves wHG 1410 progress and wHG 1430 availability"); - vector cluster_3_topology = + vector cluster_3_topology = topology_with_reader_pair(state.cluster_3, "SWITCHOVER_INITIATED"); if (sim.topology_update(state.cluster_3.get_endpoints(), cluster_3_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED for BGD wHG 1430"); @@ -380,7 +380,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta * - Verify that cluster 1 stops probing its removed target while clusters 2 * and 3 keep their status, placement, metadata target, and TLS value. */ -int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_independent_config_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { if (configure_read_only_values(sim, state.cluster_1_b) != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for cluster 1 deployment B"); return EXIT_FAILURE; @@ -398,7 +398,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt return EXIT_FAILURE; } - vector topology = + vector topology = topology_with_reader_pair(state.cluster_1_b, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1_b.get_endpoints(), topology) != EXIT_SUCCESS) { diag("Error: failed to publish deployment B topology for BGD wHG 1410"); @@ -406,7 +406,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_1_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: refreshed BGD wHG 1410 did not probe its TLS deployment B green writer"); @@ -423,7 +423,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_2.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not continue probing its TLS green writer"); @@ -432,7 +432,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_3.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not continue probing its plaintext green writer"); @@ -457,7 +457,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp index 0d375e13a1..c45fce02c6 100644 --- a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp +++ b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp @@ -32,7 +32,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -54,7 +54,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -72,8 +72,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -95,8 +95,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -134,7 +134,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) * - Publish target-only SWITCHOVER_COMPLETED. * - Verify BGD status READER_SWITCHOVER_IN_PROGRESS. */ -int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -144,7 +144,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, progress); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1360"); @@ -168,7 +168,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1360"); @@ -194,7 +194,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes * - Verify that blue-writer metadata follows the table check. * - Verify BGD status returns to READER_SWITCHOVER_IN_PROGRESS. */ -int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_config_refresh_after_writer_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -218,14 +218,14 @@ int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& } auto [table_rc, table] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (table_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not start with a blue-writer table check for wHG 1360"); return EXIT_FAILURE; } auto [blue_rc, blue] = sim.wait_for_probe_log( - table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not probe blue-writer metadata for wHG 1360"); @@ -251,7 +251,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp index 2d01d467b4..231401788a 100644 --- a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector admin_owned_endpoints { admin_owned.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -227,7 +227,7 @@ rc_t> green_server_snapshot(MYSQL* admin, const string& ta * - Disable automatic discovery and load explicit hostgroups 890-893. * - Verify explicit values replace the automatic row and persist. */ -int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_to_explicit(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.conversion; BGD_Hostgroups& hg = state.conversion_hg; @@ -237,7 +237,7 @@ int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.conversion_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 890"); @@ -372,7 +372,7 @@ int test_persistent_row_validation(MYSQL* admin, TestState& state) { * - Execute SAVE MYSQL SERVERS FROM RUNTIME. * - Verify SAVE restores wHG 920 and skips auto-generated wHG 930. */ -int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_save_from_runtime(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& explicit_cluster = state.explicit_save; RDS_BGD_Cluster& automatic_cluster = state.automatic_save; BGD_Hostgroups& explicit_hg = state.explicit_save_hg; @@ -390,14 +390,14 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat return EXIT_FAILURE; } - vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); + vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); int explicit_topology_rc = sim.topology_update(state.explicit_save_endpoints, explicit_topology); if (explicit_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 920"); return EXIT_FAILURE; } - vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); + vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); int automatic_topology_rc = sim.topology_update(state.automatic_save_endpoints, automatic_topology); if (automatic_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 930"); @@ -491,7 +491,7 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Enable automatic discovery and publish AVAILABLE topology. * - Verify the BGD row and green-server status remain unchanged. */ -int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_admin_server_status_preserved(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.admin_owned; BGD_Hostgroups& hg = state.admin_owned_hg; @@ -571,7 +571,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.admin_owned_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology beside administrator-owned wHG 1310"); @@ -579,7 +579,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not probe beside administrator-owned wHG 1310"); return EXIT_FAILURE; @@ -609,7 +609,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp index 07c7327bf9..86a4f435b5 100644 --- a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp @@ -28,7 +28,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,7 +76,7 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -88,7 +88,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -123,12 +123,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -191,7 +191,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp index 2ddc6ee15f..f99db300c9 100644 --- a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp +++ b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp @@ -33,7 +33,7 @@ struct TestState { vector servers_first_endpoints { servers_first.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -55,7 +55,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -167,7 +167,7 @@ bool runtime_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Host * - Verify no table-check probe starts. * - Load all servers and verify AVAILABLE with explicit runtime membership. */ -int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_row_before_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.row_first; BGD_Hostgroups& hg = state.row_first_hg; @@ -177,7 +177,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.row_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 840-843"); @@ -224,7 +224,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading the blue writer did not start the wHG 840 table check"); return EXIT_FAILURE; @@ -249,7 +249,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no table-check probe starts. * - Load the explicit row and verify AVAILABLE with explicit membership. */ -int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_servers_before_bgd_row(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.servers_first; BGD_Hostgroups& hg = state.servers_first_hg; @@ -259,7 +259,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.servers_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 850-853"); @@ -306,7 +306,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading wHG 850 did not start the blue table check"); return EXIT_FAILURE; @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp index 08f1e2bfc4..cf0ab931cb 100644 --- a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector after_worker_start_endpoints { after_worker_start.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -175,7 +175,7 @@ bool runtime_green_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BG * - Publish AVAILABLE topology after all membership exists. * - Verify runtime hostgroups 862 and 863 contain the configured green set. */ -int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_before_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_available; BGD_Hostgroups& hg = state.before_available_hg; @@ -209,7 +209,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.before_available_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 860"); @@ -235,7 +235,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify runtime_mysql_servers contains the complete green membership after * the worker observes the configuration change. */ -int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_discovery; BGD_Hostgroups& hg = state.after_discovery_hg; @@ -245,7 +245,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_discovery_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 870"); @@ -289,7 +289,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: wHG 870 did not probe the green writer after membership load"); return EXIT_FAILURE; @@ -307,7 +307,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Load green membership into hostgroups 882 and 883. * - Publish AVAILABLE and verify complete runtime green membership. */ -int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_worker_start(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_worker_start; BGD_Hostgroups& hg = state.after_worker_start_hg; @@ -348,7 +348,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [start_rc, start_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (start_rc != EXIT_SUCCESS) { diag("Error: wHG 880 did not start the blue table-check probe"); return EXIT_FAILURE; @@ -360,7 +360,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_worker_start_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 880"); @@ -383,7 +383,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp index 181915b473..0e63511f11 100644 --- a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp @@ -52,7 +52,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -74,7 +74,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -92,8 +92,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -115,8 +115,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -126,7 +126,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -250,7 +250,7 @@ bool all_pools_nonzero(vector& pools) { return true; } -int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int configure_status_matrix(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -336,8 +336,8 @@ int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& si return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, TestState& state, string status) { - vector topology = topology_with_readers(state.cluster, status); +int publish_topology(BGD_Simulator& sim, TestState& state, string status) { + vector topology = topology_with_readers(state.cluster, status); int rc = sim.topology_update(state.topology_endpoints, topology); return rc; @@ -362,7 +362,7 @@ bool snapshots_match(MYSQL* admin, TestState& state) { * - Publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. * - Verify rollback preserves every green pool and exact configured row. */ -int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { int config_rc = configure_status_matrix(cl, admin, sim, state); if (config_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -432,7 +432,7 @@ int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_S * - Require every green pool to remain nonzero immediately before cleanup. * - Delete topology rows and verify that ONLINE and SHUNNED pools drain. */ -int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_successful_cleanup_drains_non_offline(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int post_rc = publish_topology(sim, state, "SWITCHOVER_IN_POST_PROCESSING"); if (post_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1300"); @@ -446,7 +446,7 @@ int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& return EXIT_FAILURE; } - vector completed = target_only_completed(state.cluster); + vector completed = target_only_completed(state.cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1300"); @@ -517,7 +517,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp index 6e212f11b2..8ad356da90 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -81,7 +81,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -105,9 +105,9 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int wait_for_blue_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -169,7 +169,7 @@ int wait_for_green_pool_drain(MYSQL* admin, BGD_Hostgroups& hg) { * the blue writer is not demoted. * - Establish green writer/reader pools for terminal cleanup. */ -int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_completed(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -185,7 +185,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, return EXIT_FAILURE; } - vector topology = target_only_completed(cluster); + vector topology = target_only_completed(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -280,7 +280,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, * - Verify eligible green writer/reader pools are drained. * - Verify blue-writer and blue-reader placement remains available. */ -int test_completed_empty_topology(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_completed_empty_topology(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -318,7 +318,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp index 970378605e..aaf550f1c8 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp @@ -41,7 +41,7 @@ struct TestState { vector post_endpoints { post_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -63,7 +63,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -81,8 +81,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -100,7 +100,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -124,16 +124,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -266,7 +266,7 @@ int enable_bgd_worker(MYSQL* admin, BGD_Hostgroups& hg) { * - Change simulated writer/reader read_only values. * - Verify read_only monitoring is suppressed for deployment members. */ -int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -374,7 +374,7 @@ int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) * - Change the simulated blue-reader read_only value. * - Verify read_only monitoring remains suppressed after demotion. */ -int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -468,7 +468,7 @@ int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify the old blue pool drains and new connections route to green. * - Verify mapped blue readers remain under read_only suppression. */ -int test_first_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.post_cluster; BGD_Hostgroups& hg = state.post_hg; @@ -586,7 +586,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp index f6c616306b..b3898cf409 100644 --- a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -42,12 +42,12 @@ struct TestState { }; struct ProbeChain { - RDS_BGD_Probe_Log table; - RDS_BGD_Probe_Log blue; - RDS_BGD_Probe_Log green; + BGD_Probe_Log table; + BGD_Probe_Log blue; + BGD_Probe_Log green; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -69,7 +69,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { vector attribute_queries { "DELETE FROM mysql_hostgroup_attributes", "LOAD MYSQL SERVERS TO RUNTIME", @@ -96,23 +96,23 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int wait_for_probe_chain(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, +int wait_for_probe_chain(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, int blue_use_ssl, int green_use_ssl, ProbeChain& chain) { auto [table_rc, table] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); if (table_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [blue_rc, blue] = - sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); if (blue_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [green_rc, green] = - sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); + sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); if (green_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -125,9 +125,9 @@ bool probe_chain_ordered(ProbeChain& chain) { bool ordered = chain.table.sequence_id < chain.blue.sequence_id && chain.blue.sequence_id < chain.green.sequence_id && - chain.table.probe_kind == RDS_BGD_Probe_Kind::table_check && - chain.blue.probe_kind == RDS_BGD_Probe_Kind::metadata && - chain.green.probe_kind == RDS_BGD_Probe_Kind::metadata; + chain.table.probe_kind == BGD_Probe_Kind::table_check && + chain.blue.probe_kind == BGD_Probe_Kind::metadata && + chain.green.probe_kind == BGD_Probe_Kind::metadata; return ordered; } @@ -154,7 +154,7 @@ bool runtime_server_tls_matches(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, * - Verify table-check and blue metadata use the writer with TLS. * - Verify the mapped green writer metadata probe also uses TLS. */ -int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_writer_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.automatic; BGD_Hostgroups& hg = state.automatic_hg; @@ -198,7 +198,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.automatic_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish automatic AVAILABLE topology"); @@ -237,7 +237,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Publish AVAILABLE topology for the exact deployment. * - Verify the exact TARGET and TLS value are used in probe order. */ -int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_explicit_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.explicit_target; RDS_BGD_Cluster& distractor = state.distractor; BGD_Hostgroups& hg = state.explicit_target_hg; @@ -285,7 +285,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st vector topology_endpoints = state.explicit_target_endpoints; vector distractor_endpoints = distractor.get_endpoints(); topology_endpoints.insert(topology_endpoints.end(), distractor_endpoints.begin(), distractor_endpoints.end()); - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish explicit AVAILABLE topology"); @@ -331,7 +331,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st * - Publish AVAILABLE topology. * - Verify the created TARGET runtime row and metadata probe use TLS. */ -int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_created_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.created_target; BGD_Hostgroups& hg = state.created_target_hg; @@ -380,7 +380,7 @@ int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.created_target_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish created-TARGET AVAILABLE topology"); @@ -423,7 +423,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp index b274b4e9c5..3e3ef9aec8 100644 --- a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector fallback_endpoints { fallback.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { + vector rows = cluster.get_topology(status); for (size_t i = 0; i < pairs; ++i) { rows.push_back({ cluster.blue_readers[i].hostname, @@ -97,7 +97,7 @@ vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -118,7 +118,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_bgd( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { @@ -144,9 +144,9 @@ int configure_bgd( } int publish_post_processing( - RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) + BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) { - vector topology = + vector topology = topology_with_reader_pairs(cluster, "SWITCHOVER_IN_POST_PROCESSING", pairs); int rc = sim.topology_update(endpoints, topology); @@ -229,7 +229,7 @@ rc_t connect_and_echo(CommandLine& cl) { * - Route a client through hostgroup 1281 and verify that it reaches the * mapped green reader instead of the unmapped blue reader. */ -int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.matched; BGD_Hostgroups& hg = state.matched_hg; @@ -303,7 +303,7 @@ int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simula * - Publish SWITCHOVER_IN_POST_PROCESSING topology without reader pairs. * - Verify that the blue writer is not added to reader hostgroup 1291. */ -int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_offline_blue_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -350,7 +350,7 @@ int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Connect through hostgroup 1291 and verify routing reaches the green writer * IP pinned for the blue writer hostname. */ -int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -394,7 +394,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp index 3d10a06c50..e3f0185ed7 100644 --- a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -89,8 +89,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows { +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows { { cluster.green_writer.hostname, cluster.green_writer.hostname, @@ -145,11 +145,11 @@ bool green_rows_online(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& h return rows_online; } -int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int advance_to_post_processing(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector available = topology_with_reader_pair(cluster, "AVAILABLE"); + vector available = topology_with_reader_pair(cluster, "AVAILABLE"); int available_rc = sim.topology_update(state.topology_endpoints, available); if (available_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -160,7 +160,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int progress_rc = sim.topology_update(state.topology_endpoints, in_progress); if (progress_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -171,7 +171,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int post_rc = sim.topology_update(state.topology_endpoints, post_processing); if (post_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -193,7 +193,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Publish target-only SWITCHOVER_COMPLETED twice. * - Verify READER_SWITCHOVER_IN_PROGRESS and retained green rows. */ -int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -259,7 +259,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -283,7 +283,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector repeated_completed = target_only_completed(cluster); + vector repeated_completed = target_only_completed(cluster); int repeat_rc = sim.topology_update(state.topology_endpoints, repeated_completed); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat target-only SWITCHOVER_COMPLETED topology"); @@ -291,7 +291,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: BGD did not observe repeated target-only SWITCHOVER_COMPLETED topology"); return EXIT_FAILURE; @@ -316,7 +316,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si * - Verify metadata probing returns from the green pin to the blue writer. * - Verify green pools drain while configured green rows remain ONLINE. */ -int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -352,14 +352,14 @@ int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(unmatched_reader_online, "reader cleanup restores the unmatched blue reader in hostgroup 981"); auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: reader cleanup did not observe empty topology through the green pin"); return EXIT_FAILURE; } auto [blue_probe_rc, blue_probe] = - sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not return to the blue writer after reader cleanup"); return EXIT_FAILURE; @@ -387,7 +387,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp index dbfa042c38..589edd1291 100644 --- a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp @@ -29,7 +29,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -51,7 +51,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -86,7 +86,7 @@ int wait_for_bgd_row_absent(MYSQL* admin, BGD_Hostgroups& hg) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -98,7 +98,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -133,12 +133,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -214,7 +214,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp index b1852dedf8..a066cf5ea6 100644 --- a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp +++ b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp @@ -36,7 +36,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,8 +95,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -196,9 +196,9 @@ int replace_green_membership( } int publish_writer_lifecycle( - MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) + MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); if (sim.topology_update(cluster.get_endpoints(), initiated) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -209,7 +209,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(cluster.get_endpoints(), progress) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -220,7 +220,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(cluster.get_endpoints(), post) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -235,9 +235,9 @@ int publish_writer_lifecycle( return EXIT_SUCCESS; } -int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, +int publish_reader_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); if (sim.topology_update(cluster.get_endpoints(), completed) != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -275,7 +275,7 @@ int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Complete writer and reader switchover, then publish empty topology. * - Verify NONE and baseline blue-writer placement. */ -int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_a(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment = state.deployment_a; BGD_Hostgroups& hg = state.hostgroups; @@ -285,7 +285,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment, "AVAILABLE"); int topology_rc = sim.topology_update(deployment.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment A"); @@ -332,7 +332,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Complete deployment B and verify that routing uses deployment B without * recreating deployment A rows. */ -int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_b(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment_a = state.deployment_a; RDS_BGD_Cluster& deployment_b = state.deployment_b; BGD_Hostgroups& hg = state.hostgroups; @@ -355,7 +355,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); int topology_rc = sim.topology_update(deployment_b.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment B"); @@ -372,7 +372,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes ok(membership_matches, "runtime_mysql_servers contains only TLS-enabled deployment B green rows"); auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, deployment_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, deployment_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: deployment B green writer did not receive a TLS metadata probe"); @@ -416,7 +416,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_rollback-t.cpp b/test/tap/tests/test_rds_bgd_rollback-t.cpp index 1e1bbe270f..0c0ec1e4c6 100644 --- a/test/tap/tests/test_rds_bgd_rollback-t.cpp +++ b/test/tap/tests/test_rds_bgd_rollback-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector progress_endpoints { progress_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -82,8 +82,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -101,7 +101,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -121,16 +121,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -263,7 +263,7 @@ bool green_rows_match(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster * - Verify blue-writer placement and normal read_only processing are restored. * - Repeat AVAILABLE and verify the monitor-created green writer remains. */ -int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_initiated_rollback(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -407,7 +407,7 @@ int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta * - Verify explicit green rows and pools remain unchanged. * - Repeat AVAILABLE and verify rollback remains stable. */ -int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -600,7 +600,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp index 8b72573e6b..f87f101274 100644 --- a/test/tap/tests/test_rds_bgd_smoke-t.cpp +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -28,7 +28,7 @@ struct TestState { uint64_t probe_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -109,7 +109,7 @@ int configure_explicit_bgd(MYSQL* admin, TestState& state) { * - Record the probe sequence before publishing topology. * - Publish AVAILABLE topology to the blue and green writer endpoints. */ -int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { +int publish_available_topology(BGD_Simulator& sim, TestState& state) { int writer_rc = bgd_set_writer_read_only_0(sim, state.cluster); if (writer_rc != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated writers"); @@ -123,7 +123,7 @@ int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { } state.probe_sequence = seq; - vector topology = state.cluster.get_topology("AVAILABLE"); + vector topology = state.cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -164,10 +164,10 @@ int configure_bgd_available(MYSQL* admin, TestState& state) { * - Wait for a metadata probe after the topology publication sequence. * - Require the probe on the green writer IP without TLS. */ -int test_plaintext_green_writer_probe(RDS_BGD_Simulator& sim, TestState& state) { +int test_plaintext_green_writer_probe(BGD_Simulator& sim, TestState& state) { auto [probe_rc, probe] = sim.wait_for_probe_log( state.probe_sequence, state.cluster.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive a plaintext metadata probe"); @@ -183,7 +183,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp index 1c938b306f..1f0f6c67d1 100644 --- a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp @@ -44,7 +44,7 @@ struct TestState { vector absent_reader_endpoints { absent_reader.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -66,7 +66,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -84,8 +84,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -103,8 +103,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -114,7 +114,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -153,7 +153,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -173,14 +173,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -222,7 +222,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -249,7 +249,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -286,7 +286,7 @@ int disable_bgd(MYSQL* admin, BGD_Hostgroups& hg) { * - Verify BGD status NONE, restored blue-writer placement, and metadata * telemetry from the pinned green writer. */ -int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_before; BGD_Hostgroups& hg = state.empty_before_hg; @@ -324,7 +324,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "empty topology restores the blue writer and sets BGD status for wHG 1100 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1100 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -348,7 +348,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - Verify BGD status NONE, restored blue-writer placement, and a new * blue-writer table-check probe. */ -int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_before; BGD_Hostgroups& hg = state.absent_before_hg; @@ -386,7 +386,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat ok(true, "absent topology restores the blue writer and sets BGD status for wHG 1110 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1110 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -410,7 +410,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and metadata telemetry from the pinned green writer. */ -int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_reader; BGD_Hostgroups& hg = state.empty_reader_hg; @@ -445,7 +445,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te "empty topology completes reader cleanup for wHG 1120 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1120 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -469,7 +469,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and a new blue-writer table-check probe. */ -int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_reader; BGD_Hostgroups& hg = state.absent_reader_hg; @@ -504,7 +504,7 @@ int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, T "absent topology completes reader cleanup for wHG 1130 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1130 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -519,7 +519,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp index 06bc414ffe..cfecaec568 100644 --- a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector generic_error_endpoints { generic_error.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,8 +79,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -98,8 +98,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -109,7 +109,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -148,7 +148,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -168,14 +168,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -217,7 +217,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -244,7 +244,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -260,8 +260,8 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, - int error_number, string error_message, RDS_BGD_Probe_Log& probe) +int wait_for_metadata_error(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int error_number, string error_message, BGD_Probe_Log& probe) { vector green_endpoint { cluster.green_writer.endpoint() }; int error_rc = sim.topology_error(green_endpoint, error_number, error_message); @@ -270,7 +270,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C } auto [probe_rc, metadata_probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -279,7 +279,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C return EXIT_SUCCESS; } -int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_table_check(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { vector blue_endpoint { cluster.blue_writer.endpoint() }; int drop_rc = sim.topology_drop(blue_endpoint); if (drop_rc != EXIT_SUCCESS) { @@ -287,7 +287,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD } auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); return probe_rc; } @@ -299,7 +299,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD * - Verify BGD status NONE, restored blue-writer placement, and a subsequent * blue-writer table check. */ -int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_completion; BGD_Hostgroups& hg = state.before_completion_hg; @@ -315,7 +315,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -356,7 +356,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test * - Verify BGD status NONE, restored blue-reader routing, retained green rows, * and a subsequent blue-writer table check. */ -int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.reader_switchover; BGD_Hostgroups& hg = state.reader_switchover_hg; @@ -372,7 +372,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -410,7 +410,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si * - Verify that table checking does not restart and that the in-progress * status and blue-writer demotion remain unchanged. */ -int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_generic_metadata_error(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.generic_error; BGD_Hostgroups& hg = state.generic_error_hg; @@ -426,7 +426,7 @@ int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1105, "simulated generic metadata failure", metadata); if (metadata_rc != EXIT_SUCCESS) { diag("Error: wHG 1160 did not observe the generic metadata error on the green writer"); @@ -462,7 +462,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp index 1643200102..020c3b7869 100644 --- a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { bool topology_discovery_interval_saved { false }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -75,7 +75,7 @@ int restore_topology_discovery_interval(MYSQL* admin, TestState& state) { return rc; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -144,7 +144,7 @@ int wait_for_server_status(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, stri * - Change blue-writer `weight` and `comment`. * - Verify that the active worker does not restart with a table check. */ -int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_irrelevant_server_fields(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -156,7 +156,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -197,7 +197,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [probe_rc, probe] = sim.wait_for_probe_log( - probe_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + probe_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive the AVAILABLE metadata probe"); @@ -239,7 +239,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify that the next green-writer metadata probe uses TLS. * - Verify that discovery does not restart. */ -int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_tls_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -268,7 +268,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { // Wait for the active worker to use TLS without starting a table check. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green-writer metadata probe did not use TLS after refresh"); @@ -292,7 +292,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Delete the first green reader from hostgroup 1373. * - Verify that discovery does not restart. */ -int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_membership_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -342,7 +342,7 @@ int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Return the green writer to `ONLINE`. * - Verify that TLS metadata probing resumes without a table-check restart. */ -int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_server_eligibility_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -377,7 +377,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt // Wait for the worker to apply the refreshed server list and return to its eligible blue writer. auto [blue_probe_rc, blue_probe] = sim.wait_for_probe_log( - refresh_seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + refresh_seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: OFFLINE_SOFT refresh did not return metadata probing to the blue writer"); @@ -423,7 +423,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [probe_rc, probe] = sim.wait_for_probe_log( - online_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + online_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: ONLINE green writer did not resume TLS metadata probes"); @@ -449,7 +449,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt * - Verify that the next metadata probe occurs between 500 and 1500 milliseconds. * - Verify that the configuration refresh does not restart with a table check. */ -int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_check_interval_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -497,14 +497,14 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Consume the immediate refresh probe and the first blue probe after the worker reaches NONE. auto [first_rc, first_probe] = - sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (first_rc != EXIT_SUCCESS) { diag("Error: failed to observe the first blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; } auto [settled_rc, settled_probe] = - sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (settled_rc != EXIT_SUCCESS) { diag("Error: failed to observe the settled blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; @@ -513,7 +513,7 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Measure the steady-state interval between consecutive blue metadata probes. unsigned long long interval_start = monotonic_time(); auto [next_rc, next_probe] = sim.wait_for_probe_log( - settled_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 + settled_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 ); if (next_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not occur within 1.5 times check_interval_ms"); @@ -541,7 +541,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp index 6bd3d7ac64..84214fb22c 100644 --- a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,11 +79,11 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { +vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { RDS_BGD_Host& blue_writer = cluster.blue_readers[0]; RDS_BGD_Host& green_writer = cluster.green_readers[0]; - vector rows { + vector rows { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", "SWITCHOVER_IN_PROGRESS" }, { green_writer.hostname, green_writer.hostname, green_writer.port, @@ -103,7 +103,7 @@ vector topology_with_reader_as_writer(RDS_BGD_Cluster& clu * - Verify writer placement in hostgroup 1384. * - Verify metadata probes use TLS from green hostgroups 1385 and 1386. */ -int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_hostgroup_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; BGD_Hostgroups& refreshed_hg = state.refreshed_hostgroups; @@ -116,7 +116,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat } // Publish SWITCHOVER_IN_PROGRESS topology. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -207,7 +207,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat // Require TLS from the green writer row in refreshed green writer hostgroup 1385. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probe did not use TLS from green writer hostgroup 1385"); @@ -230,7 +230,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify that the newly mapped writer moves to hostgroup 1384. * - Verify that metadata probing uses TLS from the new green writer target. */ -int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_mapped_writer_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.refreshed_hostgroups; RDS_BGD_Host& previous_writer = cluster.blue_writer; @@ -238,7 +238,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& RDS_BGD_Host& mapped_target = cluster.green_readers[0]; // Publish topology that maps the first blue reader to the first green reader. - vector topology = topology_with_reader_as_writer(cluster); + vector topology = topology_with_reader_as_writer(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for the new mapped writer"); @@ -282,7 +282,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, mapped_target.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, mapped_target.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not use TLS from green writer hostgroup 1385"); @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp index 9bf8b8ee94..333580707c 100644 --- a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp @@ -35,7 +35,7 @@ struct TestState { int64_t reader_log_baseline { -1 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -57,7 +57,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -75,8 +75,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -94,9 +94,9 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int wait_for_green_observation(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_observation(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -211,7 +211,7 @@ int create_blue_writer_pool(CommandLine& cl, MYSQL* admin, TestState& state) { * - Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. * - Verify BGD status AVAILABLE. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -239,7 +239,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 970"); @@ -272,7 +272,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Change simulated blue writer/reader read_only values. * - Verify BGD suppresses their normal placement changes. */ -int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -282,7 +282,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology"); @@ -318,7 +318,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_INITIATED topology"); @@ -362,11 +362,11 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify the blue writer moves from hostgroup 970 to 971. * - Verify the mapped blue reader remains suppressed in hostgroup 971. */ -int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -412,7 +412,7 @@ int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify writer restoration, blue-pool drain, and green backend routing. * - Repeat POST_PROCESSING and verify the post-cutover pool is not drained. */ -int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -422,7 +422,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology"); @@ -484,7 +484,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING topology"); @@ -512,7 +512,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); From 982baeb362453530bf42427fccad4e5a2a4dae3a Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 23:46:16 +0000 Subject: [PATCH 17/26] feat: add Aurora replica simulator service --- include/MySQL_Monitor.hpp | 28 ++- include/SQLite3_Server.h | 11 - lib/MySQL_Monitor.cpp | 29 +-- lib/ProxySQL_Admin.cpp | 2 +- src/SQLite3_Server.cpp | 424 +++++++++++++------------------------- 5 files changed, 170 insertions(+), 324 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index ad599493a5..38c379d825 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -62,6 +62,26 @@ struct cmp_str { #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" #define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" #define QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" +#define QUERY_AWS_AURORA_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID," \ + "IF(" \ + "SESSION_ID = 'MASTER_SESSION_ID' AND " \ + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " \ + "'probably_former_MASTER_SESSION_ID', SESSION_ID" \ + ") SESSION_ID, " \ + "LAST_UPDATE_TIMESTAMP, " \ + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " \ + "CPU " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" \ + " ( " \ + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" \ + " OR SESSION_ID = 'MASTER_SESSION_ID'" \ + " ) " \ + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" \ + " ORDER BY SERVER_ID" +#define QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IS_CURRENT " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS ORDER BY SERVER_ID" /* @@ -77,14 +97,6 @@ A single AWS_Aurora_monitor_node will have a AWS_Aurora_status_entry per check. */ -#ifdef TEST_AURORA - -#define TEST_AURORA_MONITOR_BASE_QUERY \ - "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, REPLICA_LAG_IN_MILLISECONDS, CPU"\ - " FROM REPLICA_HOST_STATUS ORDER BY SERVER_ID " - -#endif - class AWS_Aurora_replica_host_status_entry { public: char * server_id = nullptr; diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 20b5f30ab5..eaf3cd017a 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -70,20 +70,9 @@ class SQLite3_Server { public: SQLite3DB *sessdb; #ifdef TEST_AURORA - unsigned int cur_aurora_writer[3]; - unsigned int num_aurora_servers[3]; - unsigned int max_num_aurora_servers; - pthread_mutex_t aurora_mutex; - /** - * @brief Handles queries to table 'REPLICA_HOST_STATUS'. - * @details This function needs to be called with lock on mutex aurora_mutex already acquired. - * @param sess The session which request is to be handled. - */ - void populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg); void init_aurora_ifaces_string(std::string& s); #endif // TEST_AURORA #ifdef TEST_GALERA - //unsigned int cur_aurora_writer[3]; unsigned int num_galera_servers[3]; unsigned int max_num_galera_servers; pthread_mutex_t galera_mutex; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index ff3c31389b..a76bb3b2f5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6290,12 +6290,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#ifdef TEST_AURORA - { - string query { TEST_AURORA_MONITOR_BASE_QUERY + std::to_string(wHG) }; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query.c_str()); - } -#else // for reference we list the old queries. // original implementation: // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS > 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); @@ -6306,27 +6300,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { // with: // "REPLICA_LAG_IN_MILLISECONDS >= 0" // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); - { - const char * query = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " // it seems that during a failover, the old writer can keep MASTER_SESSION_ID because not updated - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" // lag between 0 and 10 minutes - " OR SESSION_ID = 'MASTER_SESSION_ID'" // or server with MASTER_SESSION_ID - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" // ignore decommissioned or renamed nodes, see https://github.com/sysown/proxysql/issues/3484 - " ORDER BY SERVER_ID"; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); - } -#endif // TEST_AURORA + mmsd->async_exit_status = mysql_query_start( + &mmsd->interr, mmsd->mysql, QUERY_AWS_AURORA_REPLICA_HOST_STATUS); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); #ifdef DEBUG diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 9c54baf9cd..0a8dcdae95 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -9225,7 +9225,7 @@ void ProxySQL_Admin::enable_galera_testing() { void ProxySQL_Admin::enable_aurora_testing_populate_mysql_servers() { sqlite3_stmt *statement=NULL; - unsigned int num_aurora_servers = GloSQLite3Server->num_aurora_servers[0]; + constexpr unsigned int num_aurora_servers = 4; admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 1271 AND 1276"); char *query=(char *)"INSERT INTO mysql_servers (hostgroup_id,hostname,use_ssl,comment) VALUES (?1, ?2, ?3, ?4)"; auto [rc1, statement_unique] = admindb->prepare_v2(query); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 093b284045..c435577530 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -47,6 +47,20 @@ static int random_replication_lag_seconds() { } #endif +#ifdef TEST_AURORA +static std::string simulator_sql_quote(const std::string& value) { + std::string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} +#endif + #ifdef TEST_REPLICATIONLAG static void ensure_replicationlag_table_loaded(SQLite3_Server* server, MySQL_Session* sess) { if (server->replicationlag_map_size() == 0) { @@ -917,7 +931,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } delete control_result; - if (run_query && aws_bgd_table_check) { + if (run_query && aws_bgd_table_check) { const char* topology_sql = topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"; static constexpr size_t topology_sql_len = sizeof("SELECT 1") - 1; @@ -952,39 +966,117 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } #endif // TEST_AURORA || TEST_RDS_BGD - #ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - - if (strcasestr(query_no_space, TEST_AURORA_MONITOR_BASE_QUERY)) { - string s_whg { query_no_space + (sizeof(TEST_AURORA_MONITOR_BASE_QUERY) - 1) }; - uint32_t whg = atoi(s_whg.c_str()); - - GloSQLite3Server->populate_aws_aurora_table(sess, whg); - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; +#ifdef TEST_AURORA + const bool aws_aurora_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_REPLICA_HOST_STATUS) == 0; + const bool aws_aurora_bgd_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS) == 0; + if (aws_aurora_replica_query || aws_aurora_bgd_replica_query) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator could not identify the accepted backend address"); + run_query=false; + } else { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; + const int backend_port = sess->client_myds->proxy_addr.port; + const std::string predicate { + "backend_ip=" + simulator_sql_quote(backend_ip) + + " AND backend_port=" + std::to_string(backend_port) + }; + char *control_error=nullptr; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=nullptr; + const std::string control_query { + "SELECT replica_set_id,replica_table_present,error_code,error_msg " + "FROM AWS_AURORA_REPLICA_CONTROL WHERE " + predicate + }; + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + + std::string replica_set_id {}; + bool replica_table_present=false; + unsigned int configured_error=0; + std::string configured_error_msg {}; + if (control_error == nullptr && control_result && + control_result->rows_count == 1) { + SQLite3_row *row=control_result->rows.front(); + replica_set_id=row->fields[0] ? row->fields[0] : ""; + replica_table_present=atoi(row->fields[1]) != 0; + configured_error=static_cast(atoi(row->fields[2])); + configured_error_msg=row->fields[3] ? row->fields[3] : ""; + } + delete control_result; - const auto match_writer = [&whg](const aurora_hg_info_t& hg_info) { - return std::get(hg_info) == whg; - }; - const auto hg_info_it = std::find_if(hgs_info.begin(), hgs_info.end(), match_writer); - string select_query { - "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS,CPU" - " FROM REPLICA_HOST_STATUS " - }; + const std::string log_query { + "INSERT INTO AWS_AURORA_REPLICA_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,replica_set_id,encrypted) VALUES (" + + simulator_sql_quote(backend_ip) + "," + + std::to_string(backend_port) + "," + + simulator_sql_quote( + aws_aurora_replica_query ? "ordinary" : "bgd_membership") + "," + + (replica_set_id.empty() ? "NULL" : simulator_sql_quote(replica_set_id)) + + "," + (sess->client_myds->encrypted ? "1" : "0") + ")" + }; - if (hg_info_it == hgs_info.end()) { - select_query += " LIMIT 0"; + if (control_error != nullptr) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } else if (!sqlite_sess->sessdb->execute(log_query.c_str())) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to record the replica probe"); + run_query=false; + } else if (!replica_table_present) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); + run_query=false; + } else if (configured_error != 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, + static_cast(configured_error), + configured_error_msg.c_str()); + run_query=false; + } else { + const std::string set_literal { simulator_sql_quote(replica_set_id) }; + std::string select_query {}; + if (aws_aurora_replica_query) { + select_query = + "SELECT SERVER_ID," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' AND SERVER_ID<>(" + "SELECT SERVER_ID FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + set_literal + " AND SESSION_ID='MASTER_SESSION_ID' " + "ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1) " + "THEN 'probably_former_MASTER_SESSION_ID' ELSE SESSION_ID END AS SESSION_ID," + "LAST_UPDATE_TIMESTAMP," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' THEN 0 " + "ELSE REPLICA_LAG_IN_MILLISECONDS END AS REPLICA_LAG_IN_MILLISECONDS," + "CPU FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " AND ((REPLICA_LAG_IN_MILLISECONDS>=0 AND " + "REPLICA_LAG_IN_MILLISECONDS<=600000) OR " + "SESSION_ID='MASTER_SESSION_ID')" + " AND LAST_UPDATE_TIMESTAMP>datetime('now','-180 seconds')" + " ORDER BY SERVER_ID"; } else { - const string& domain_name { std::get(*hg_info_it) }; - select_query += " WHERE DOMAIN_NAME='" + domain_name + "' ORDER BY SERVER_ID"; + select_query = + "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,IS_CURRENT " + "FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " ORDER BY SERVER_ID"; } - free(query); - query = static_cast(malloc(select_query.length() + 1)); - memcpy(query, select_query.c_str(), select_query.length()); - query[select_query.length()] = '\0'; + l_free(query_length,query); + query=l_strdup(select_query.c_str()); + query_length=select_query.length()+1; } } + } #endif // TEST_AURORA #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { @@ -1096,19 +1188,8 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_GALERA) || defined(TEST_GROUPREP) if (strncasecmp("SELECT",query_no_space,6)==0) { -#ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); -#ifdef TEST_AURORA_RANDOM - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } -#endif - } -#endif // TEST_AURORA #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { pthread_mutex_unlock(&GloSQLite3Server->galera_mutex); @@ -1141,7 +1222,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_GALERA || TEST_GROUPREP sqlite3 *db = sqlite_sess->sessdb->get_db(); bool in_trans = false; if ((*proxy_sqlite3_get_autocommit)(db)==0) { @@ -1484,14 +1565,8 @@ SQLite3_Server::~SQLite3_Server() { void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { if(!s.empty()) s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes + constexpr unsigned int max_num_aurora_servers = 10; for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; for (unsigned int i=11; i(cpu_i) / 100; - - return cpu; -} - -string get_curtime_str() { - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string s = string(lut); - return s; -} - -void bind_query_params( - SQLite3DB* db, - sqlite3_stmt* stmt, - const string& server_id, - const string& domain, - const string& session_id, - float cpu, - const string& lut, - int32_t lag_ms -) { - int rc = 0; - - rc=(*proxy_sqlite3_bind_text)(stmt, 1, server_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 2, domain.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 3, session_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 4, cpu); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 5, lut.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 6, lag_ms); ASSERT_SQLITE_OK(rc, db); - SAFE_SQLITE3_STEP2(stmt); - rc=(*proxy_sqlite3_clear_bindings)(stmt); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_reset)(stmt); ASSERT_SQLITE_OK(rc, db); -} - -/** - * @brief Extracts SERVER_ID from the supplied hostname using DOMAIN_NAME. - * @param hostname The server hostname (SERVER_ID + DOMAIN_NAME)). - * @param domain_name The server DOMAIN_NAME as in 'mysql_aws_aurora_hostgroups' - * @return Either the SERVER_ID in the supplied hostname or empty if DOMAIN_NAME failed to match. - */ -string get_server_id(const string& hostname, const string& domain_name) { - string::size_type pos = hostname.find(domain_name); - - if (pos == string::npos) { - return {}; - } else { - return hostname.substr(0, pos); - } -} - -void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg) { - int rc = 0; - sqlite3_stmt* stmt = NULL; - const char query[] { "INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5, ?6)" }; - - auto [stmt_rc, stmt_unique] = sessdb->prepare_v2(query); - rc = stmt_rc; - stmt = stmt_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - -#ifndef TEST_AURORA_RANDOM - SQLite3_result* host_status = NULL; - - { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - - string query { - "SELECT SERVER_ID,DOMAIN_NAME,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS" - " FROM REPLICA_HOST_STATUS" - }; - sessdb->execute_statement(query.c_str(), &error, &cols, &affected_rows, &host_status); - } - - // If empty, we fill the map with sensible defaults for performing manual testing. - if (host_status->rows.empty()) { - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; - SQLite3_result* resultset = nullptr; - - { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - - GloAdmin->admindb->execute_statement( - "SELECT hostname, hostgroup_id FROM mysql_servers WHERE hostgroup_id BETWEEN 1270 AND 1300" - " GROUP BY HOSTNAME", - &error, &cols, &affected_rows, &resultset - ); - } - - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - vector proc_srvs {}; - - for (const aurora_hg_info_t& hg_info : hgs_info) { - const auto match_writer = [&hg_info](const SQLite3_row* row) { - return atoi(row->fields[1]) == std::get(hg_info); - }; - const auto mysrv_it = std::find_if(resultset->rows.begin(), resultset->rows.end(), match_writer); - bool writer_set = false; - - for (const SQLite3_row* r : resultset->rows) { - const string srv_hostname { r->fields[0] }; - const uint32_t srv_hg_id = atoi(r->fields[1]); - const string& aurora_domain { std::get(hg_info) }; - - if ( - srv_hostname.find(aurora_domain) == string::npos || - std::find(proc_srvs.begin(), proc_srvs.end(), srv_hostname) != proc_srvs.end() - ) { - continue; - } - - const string server_id { - get_server_id(srv_hostname, std::get(hg_info)) - }; - - string session_id {}; - - if ( - (mysrv_it == resultset->rows.end() && writer_set == false) || - (srv_hg_id == std::get(hg_info) && writer_set == false) - ) { - session_id = "MASTER_SESSION_ID"; - writer_set = true; - } else { - session_id = "TESTID-" + server_id + aurora_domain + "-R"; - } - - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - const int lag_ms = 0; - - bind_query_params(sessdb, stmt, server_id, aurora_domain, session_id, cpu, lut, lag_ms); - proc_srvs.push_back(srv_hostname); - } - } - - delete resultset; - } else { - // We just re-generate deterministic 'SESSION_IDS', preserving 'MASTER_SESSION_ID' values: - // 'SESSION_IDS' are preserved, 'MASTER_SESSION_ID' or others. - for (SQLite3_row* row : host_status->rows) { - const char* server_id = row->fields[0]; - const char* domain_name = row->fields[1]; - - const char update_query_t[] { - "UPDATE REPLICA_HOST_STATUS SET SESSION_ID='%s',CPU=%f,LAST_UPDATE_TIMESTAMP='%s'" - " WHERE SERVER_ID='%s' AND DOMAIN_NAME='%s' AND SESSION_ID!='MASTER_SESSION_ID'" - }; - - const string session_id { "TESTID-" + string { server_id } + domain_name + "-R" }; - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - - const string update_query { - cstr_format(update_query_t, session_id.c_str(), cpu, lut.c_str(), server_id, domain_name).str - }; - - sessdb->execute(update_query.c_str()); - } - } - - delete host_status; -#else - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - - string lut { get_curtime_str() }; - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; i Date: Thu, 13 Aug 2026 23:46:25 +0000 Subject: [PATCH 18/26] test: add Aurora BGD simulator controls --- test/tap/tap/Makefile | 7 +- test/tap/tap/aurora_bgd_simulator.cpp | 40 +++++ test/tap/tap/aurora_bgd_simulator.h | 52 +++++++ test/tap/tap/bgd_simulator.cpp | 206 ++++++++++++++++++++++++++ test/tap/tap/bgd_simulator.h | 45 +++++- 5 files changed, 347 insertions(+), 3 deletions(-) create mode 100644 test/tap/tap/aurora_bgd_simulator.cpp create mode 100644 test/tap/tap/aurora_bgd_simulator.h diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index 6e090f4ad9..4ceb4b1ca5 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -86,11 +86,14 @@ cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) +aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h + $(CXX) -fPIC -c aurora_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/aurora_bgd_simulator.cpp b/test/tap/tap/aurora_bgd_simulator.cpp new file mode 100644 index 0000000000..1fb37f78c6 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.cpp @@ -0,0 +1,40 @@ +#include "aurora_bgd_simulator.h" + +using namespace std; + +Endpoint Aurora_BGD_Endpoint::backend() { + return { ip, port }; +} + +Endpoint Aurora_BGD_Endpoint::host_endpoint() { + return { hostname, port }; +} + +Aurora_Replica_Row Aurora_BGD_Member::replica_row() { + return { + server_id, + session_id, + cpu, + last_update_timestamp, + replica_lag_in_milliseconds, + is_current, + }; +} + +vector Aurora_BGD_Membership_Set::replica_rows() { + vector rows {}; + rows.reserve(members.size()); + for (Aurora_BGD_Member& member : members) { + rows.push_back(member.replica_row()); + } + return rows; +} + +vector Aurora_BGD_Membership_Set::backends() { + vector endpoints {}; + endpoints.reserve(serving_endpoints.size()); + for (Aurora_BGD_Endpoint& endpoint : serving_endpoints) { + endpoints.push_back(endpoint.backend()); + } + return endpoints; +} diff --git a/test/tap/tap/aurora_bgd_simulator.h b/test/tap/tap/aurora_bgd_simulator.h new file mode 100644 index 0000000000..6df8f01db2 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.h @@ -0,0 +1,52 @@ +#ifndef TAP_AURORA_BGD_SIMULATOR_H +#define TAP_AURORA_BGD_SIMULATOR_H + +#include +#include + +#include "bgd_simulator.h" + +using namespace std; + +/** An AWS-style Aurora hostname and its fixed simulator backend address. */ +struct Aurora_BGD_Endpoint { + string hostname; + string ip; + int port; + + Endpoint backend(); + Endpoint host_endpoint(); +}; + +/** One Aurora cluster member published through REPLICA_HOST_STATUS. */ +struct Aurora_BGD_Member { + string server_id; + string session_id; + Aurora_BGD_Endpoint endpoint; + double cpu { 0 }; + string last_update_timestamp; + double replica_lag_in_milliseconds { 0 }; + bool is_current { true }; + + Aurora_Replica_Row replica_row(); +}; + +/** A membership snapshot and every backend allowed to return it. */ +struct Aurora_BGD_Membership_Set { + string replica_set_id; + vector members; + vector serving_endpoints; + + vector replica_rows(); + vector backends(); +}; + +/** Stable identity used to describe an AWS member rename during switchover. */ +struct Aurora_BGD_Rename { + string production_server_id; + string target_server_id; + string session_id; + string cached_target_ip; +}; + +#endif // TAP_AURORA_BGD_SIMULATOR_H diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp index c0f476e706..8c0641fa24 100644 --- a/test/tap/tap/bgd_simulator.cpp +++ b/test/tap/tap/bgd_simulator.cpp @@ -25,6 +25,20 @@ rc_t parse_probe_kind(string value) { return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; } +const char* replica_probe_kind_string(Aurora_Replica_Probe_Kind kind) { + return kind == Aurora_Replica_Probe_Kind::ordinary ? "ordinary" : "bgd_membership"; +} + +rc_t parse_replica_probe_kind(string value) { + if (value == "ordinary") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::ordinary }; + } + if (value == "bgd_membership") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::bgd_membership }; + } + return { EXIT_FAILURE, Aurora_Replica_Probe_Kind::ordinary }; +} + } // namespace int BGD_Simulator::topology_update(vector backends, vector rows) { @@ -99,6 +113,84 @@ int BGD_Simulator::topology_error(vector backends, int error_code, str return execute_transaction(statements); } +int BGD_Simulator::replica_update( + string replica_set_id, + vector rows, + vector backends) +{ + if (replica_set_id.empty() || backends.empty()) { + return EXIT_FAILURE; + } + + vector statements { + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + }; + for (Aurora_Replica_Row& row : rows) { + statements.push_back( + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + sql_quote(replica_set_id) + "," + sql_quote(row.server_id) + "," + + sql_quote(row.session_id) + "," + to_string(row.cpu) + "," + + sql_quote(row.last_update_timestamp) + "," + + to_string(row.replica_lag_in_milliseconds) + "," + + (row.is_current ? "1" : "0") + ")"); + } + for (Endpoint& backend : backends) { + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + sql_quote(replica_set_id) + ",1,0,'')"); + } + + return execute_transaction(statements); +} + +int BGD_Simulator::replica_delete(string replica_set_id) { + if (replica_set_id.empty()) { + return EXIT_FAILURE; + } + vector statements { + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + }; + return execute_transaction(statements); +} + +int BGD_Simulator::replica_drop(vector backends) { + return replica_error( + backends, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); +} + +int BGD_Simulator::replica_error( + vector backends, int error_code, string error_msg) +{ + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + const bool table_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + const string predicate { backend_predicate(backend) }; + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + "COALESCE((SELECT replica_set_id FROM AWS_AURORA_REPLICA_CONTROL WHERE " + + predicate + "),'')," + (table_present ? "1" : "0") + "," + + to_string(error_code) + "," + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + int BGD_Simulator::cleanup() { vector statements { "DELETE FROM READONLY_STATUS", @@ -106,6 +198,24 @@ int BGD_Simulator::cleanup() { "DELETE FROM AWS_BGD_CONTROL", "DELETE FROM AWS_BGD_PROBE_LOG", }; + + if (connection() == nullptr) { + return EXIT_FAILURE; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT name FROM sqlite_master WHERE type='table' AND name IN (" + "'AWS_AURORA_REPLICA_CONTROL','AWS_AURORA_REPLICA_PROBE_LOG'," + "'REPLICA_HOST_STATUS')"); + if (rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (mysql_res_row& row : rows) { + if (row.size() != 1) { + return EXIT_FAILURE; + } + statements.push_back("DELETE FROM " + row.front()); + } return execute_transaction(statements); } @@ -200,6 +310,102 @@ rc_t BGD_Simulator::wait_for_probe_log( return { ETIMEDOUT, {} }; } +rc_t BGD_Simulator::replica_probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::replica_probe_log_since( + uint64_t sequence_id) +{ + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind," + "COALESCE(replica_set_id,''),encrypted " + "FROM AWS_AURORA_REPLICA_PROBE_LOG WHERE sequence_id>" + + to_string(sequence_id) + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 6) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, probe_kind] = parse_replica_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + probe_kind, + row[4], + atoi(row[5].c_str()) != 0, + }); + } + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted, + string replica_set_id) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (Aurora_Replica_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (replica_set_id.empty() || log.replica_set_id == replica_set_id) && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (Aurora_Replica_Probe_Log& log : logs) { + diag( + "Observed Aurora replica probe sequence=%llu backend=%s:%d kind=%s set=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + replica_probe_kind_string(log.probe_kind), + log.replica_set_id.c_str(), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for Aurora replica probe backend=%s:%d kind=%s set=%s encrypted=%d", + backend.host.c_str(), backend.port, replica_probe_kind_string(probe_kind), + replica_set_id.c_str(), encrypted); + return { ETIMEDOUT, {} }; +} + int BGD_Simulator::execute_transaction(vector& statements) { if (execute("START TRANSACTION") != EXIT_SUCCESS) { return EXIT_FAILURE; diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h index a98013319d..30650da764 100644 --- a/test/tap/tap/bgd_simulator.h +++ b/test/tap/tap/bgd_simulator.h @@ -33,13 +33,46 @@ struct BGD_Probe_Log { bool encrypted; }; -/** Controls shared AWS BGD topology responses and reads their probe log. */ +/** One row returned by the simulated Aurora replica-status service. */ +struct Aurora_Replica_Row { + string server_id; + string session_id; + double cpu; + string last_update_timestamp; + double replica_lag_in_milliseconds; + bool is_current; +}; + +/** Identifies which Aurora replica-status query reached the simulator. */ +enum class Aurora_Replica_Probe_Kind { + ordinary, + bgd_membership, +}; + +/** One Aurora replica-status query observed by the shared AWS simulator. */ +struct Aurora_Replica_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + Aurora_Replica_Probe_Kind probe_kind; + string replica_set_id; + bool encrypted; +}; + +/** Controls shared AWS BGD topology and Aurora replica simulator responses. */ class BGD_Simulator : public Cluster_Simulator { public: int topology_update(vector backends, vector rows); int topology_delete(vector backends); int topology_drop(vector backends); int topology_error(vector backends, int error_code, string error_msg); + int replica_update( + string replica_set_id, + vector rows, + vector backends + ); + int replica_delete(string replica_set_id); + int replica_drop(vector backends); + int replica_error(vector backends, int error_code, string error_msg); int cleanup(); rc_t probe_log_last_sequence(); @@ -51,6 +84,16 @@ class BGD_Simulator : public Cluster_Simulator { uint32_t timeout_ms, int encrypted = -1 ); + rc_t replica_probe_log_last_sequence(); + rc_t> replica_probe_log_since(uint64_t sequence_id); + rc_t wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1, + string replica_set_id = "" + ); protected: static string backend_predicate(Endpoint backend); From 1c3702c821a270103a165e6509e9d4ee4323f6af Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 23:46:35 +0000 Subject: [PATCH 19/26] test: migrate Aurora scenarios to replica sets --- .../cluster_simulator/cluster_simulator.cpp | 6 +- .../cluster_simulator/lib/aurora_utils.cpp | 207 +++++++++++++----- .../deps/cluster_simulator/lib/aurora_utils.h | 6 +- test/tap/groups/cluster_sim_aurora/add-hosts | 28 +++ 4 files changed, 184 insertions(+), 63 deletions(-) diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index 2ee974de1a..cced5822a4 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1798,10 +1798,8 @@ std::pair simulate_aws_aurora_cluster_state( const auto& aurora_new_state_to_set { aurora_update_cluster_state(aurora_init_servers_state, aurora_new_servers_state) }; - // cleanup=1 deletes REPLICA_HOST_STATUS rows whose SERVER_ID is not in the new - // state, so payloads can simulate a server being removed from the topology - // (e.g. for autopurge_missing_checks coverage). Backwards-compatible: existing - // payloads keep the full server list in new_state, so nothing extra is purged. + // Replace the published replica sets and their complete backend mappings so + // payloads can simulate members being removed (for example, autopurge coverage). prep_aurora_final_state_err = prepare_aurora_cluster_state(proxysql_sqlite, aurora_new_servers_state, 1); diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index 920acd30d9..68ea1d4c86 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -1,10 +1,14 @@ #include "aurora_utils.h" #include +#include +#include #include +#include #include #include #include +#include // NOTE: Only needed during testing #include @@ -354,89 +358,176 @@ std::pair prepare_mysql_aurora_hostgroups( return { EXIT_SUCCESS, "" }; } -const char t_aurora_server_state_insert[] { - "INSERT OR REPLACE INTO REPLICA_HOST_STATUS(" - " SERVER_ID," - " DOMAIN_NAME," - " SESSION_ID," - " CPU," - " LAST_UPDATE_TIMESTAMP," - " REPLICA_LAG_IN_MILLISECONDS" - ") VALUES (" - "'%s', '%s', '%s', %d, '%s', %d" - ")" -}; +namespace { -std::pair prepare_aurora_cluster_state( - MYSQL* proxysql_sqlite, - const vector& servers, - uint32_t cleanup -) { - int query_error = 0; +string aurora_sql_quote(const string& value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} - if (cleanup) { - string srv_ids {}; - string domain_names {}; +std::pair load_aurora_backend_addresses( + std::unordered_map& addresses) +{ + const char* host_file_path = getenv("CLUSTER_SIM_HOST_FILE"); + if (host_file_path == nullptr || *host_file_path == '\0') { + return { EXIT_FAILURE, "CLUSTER_SIM_HOST_FILE is not configured" }; + } - for (const auto& server : servers) { - srv_ids += "'" + std::get(server) + "'"; - domain_names += "'" + std::get(server) + "'"; + std::ifstream host_file { host_file_path }; + if (!host_file.is_open()) { + return { + EXIT_FAILURE, + "Unable to open CLUSTER_SIM_HOST_FILE '" + string { host_file_path } + "'" + }; + } - if (&server != &servers.back()) { - srv_ids += ","; - domain_names += ","; - } + string line {}; + uint64_t line_number = 0; + while (std::getline(host_file, line)) { + ++line_number; + std::istringstream fields { line }; + string hostname {}; + string ip {}; + if (!(fields >> hostname) || hostname.front() == '#') { + continue; } - - string cleanup_query {}; - - if (cleanup == 1) { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS WHERE SERVER_ID NOT IN (" + - srv_ids + ") OR DOMAIN_NAME NOT IN (" + domain_names + ")"; - } else { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS"; + if (!(fields >> ip)) { + return { + EXIT_FAILURE, + "Missing IP in CLUSTER_SIM_HOST_FILE at line " + std::to_string(line_number) + }; } - query_error = mysql_query(proxysql_sqlite, cleanup_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, cleanup_query, __FILE__, __LINE__); + auto existing = addresses.find(hostname); + if (existing != addresses.end() && existing->second != ip) { + return { + EXIT_FAILURE, + "Conflicting CLUSTER_SIM_HOST_FILE mappings for '" + hostname + "'" + }; } + addresses[hostname] = ip; } - usleep(1000 * 1000); + return { EXIT_SUCCESS, "" }; +} + +} // namespace - // NOTE: We adquire a 'write lock' so there are no dirty reads on ProxySQL side - // while we write the new values. - query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); +std::pair prepare_aurora_cluster_state( + MYSQL* proxysql_sqlite, + const vector& servers, + uint32_t cleanup +) { + std::unordered_map backend_addresses {}; + auto [host_file_rc, host_file_error] = + load_aurora_backend_addresses(backend_addresses); + if (host_file_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, host_file_error }; + } + + std::map> replica_sets {}; + for (const aurora_server_state_t& server : servers) { + replica_sets[std::get(server)].push_back(server); + } + + int query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); if (query_error) { return create_query_error(proxysql_sqlite, "BEGIN IMMEDIATE", __FILE__, __LINE__); } - for (const auto& server : servers) { - string server_insert_query {}; + const auto execute_or_rollback = [proxysql_sqlite](const string& query) { + if (mysql_query(proxysql_sqlite, query.c_str()) == 0) { + return std::pair { EXIT_SUCCESS, "" }; + } + auto error = create_query_error(proxysql_sqlite, query, __FILE__, __LINE__); + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return error; + }; - string_format( - t_aurora_server_state_insert, - server_insert_query, - std::get(server).c_str(), - std::get(server).c_str(), - std::get(server).c_str(), - 0, - "", - std::get(server) - ); + if (cleanup) { + auto [control_rc, control_error] = + execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else { + for (const auto& replica_set : replica_sets) { + const string set_literal { aurora_sql_quote(replica_set.first) }; + auto [control_rc, control_error] = execute_or_rollback( + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + set_literal); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback( + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } + } - query_error = mysql_query(proxysql_sqlite, server_insert_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, server_insert_query, __FILE__, __LINE__); + const string timestamp { get_fmt_time() }; + for (const auto& replica_set : replica_sets) { + const string& replica_set_id = replica_set.first; + std::map, bool> mapped_backends {}; + for (const aurora_server_state_t& server : replica_set.second) { + const string& server_id = std::get(server); + string session_id = std::get(server); + if (session_id.empty()) { + session_id = "TESTID-" + server_id + replica_set_id + "-R"; + } + const string hostname { server_id + replica_set_id }; + auto address = backend_addresses.find(hostname); + if (address == backend_addresses.end()) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { + EXIT_FAILURE, + "Missing CLUSTER_SIM_HOST_FILE mapping for Aurora member '" + + hostname + "'" + }; + } + + const string row_query { + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + aurora_sql_quote(replica_set_id) + "," + aurora_sql_quote(server_id) + + "," + aurora_sql_quote(session_id) + ",0," + + aurora_sql_quote(timestamp) + "," + + std::to_string(std::get(server)) + + ",1)" + }; + auto [row_rc, row_error] = execute_or_rollback(row_query); + if (row_rc != EXIT_SUCCESS) return { row_rc, row_error }; + + mapped_backends[{ address->second, 3306 }] = true; + } + + for (const auto& backend : mapped_backends) { + const string control_query { + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + aurora_sql_quote(backend.first.first) + "," + + std::to_string(backend.first.second) + "," + aurora_sql_quote(replica_set_id) + + ",1,0,'')" + }; + auto [control_rc, control_error] = execute_or_rollback(control_query); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; } } query_error = mysql_query(proxysql_sqlite, "COMMIT"); if (query_error) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } + // Allow an already scheduled monitor probe to observe the committed snapshot. + usleep(1000 * 1000); + return { EXIT_SUCCESS, "" }; } diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index 1c75495abf..f8ba8dd2e3 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -61,6 +61,11 @@ std::pair prepare_mysql_aurora_hostgroups( const std::vector& hostgroups_configs ); +/** + * Publishes ordinary Aurora JSON state through backend-address replica sets. + * DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves every + * SERVER_ID + DOMAIN_NAME member hostname to its simulated backend address. + */ std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const std::vector& servers, @@ -94,4 +99,3 @@ std::vector aurora_update_cluster_state( ); #endif - diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 77d02bd058..94b8d811c7 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -28,6 +28,11 @@ host.2.18.cluster2.aws.test 127.0.2.18 host.2.19.cluster2.aws.test 127.0.2.19 host.2.20.cluster2.aws.test 127.0.2.20 +# Cluster 2 members reused with the first test domain. +host.2.11.aws-test.com 127.0.2.11 +host.2.12.aws-test.com 127.0.2.12 +host.2.13.aws-test.com 127.0.2.13 + # Cluster 3 host.3.11.aws.3.test.com 127.0.3.11 host.3.12.aws.3.test.com 127.0.3.12 @@ -39,3 +44,26 @@ host.3.17.aws.3.test.com 127.0.3.17 host.3.18.aws.3.test.com 127.0.3.18 host.3.19.aws.3.test.com 127.0.3.19 host.3.20.aws.3.test.com 127.0.3.20 + +# Aurora blue/green deployment A: production members and target resources. +aurora-a-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.11 +aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.12 +aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.13 +aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com 127.0.11.20 +aurora-a-green-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-green-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Canonical names observed after target-member rename retain target addresses. +aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-canonical-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-canonical-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Aurora blue/green deployment B supports repeated and concurrent scenarios. +aurora-b-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.11 +aurora-b-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.12 +aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com 127.0.12.20 +aurora-b-green-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-green-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 From 5f6ed806b1b19dc0e615748a2e7f9969d5e27d80 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 00:10:25 +0000 Subject: [PATCH 20/26] fix: harden Aurora simulator publication --- src/SQLite3_Server.cpp | 40 +- .../cluster_simulator/cluster_simulator.cpp | 2 +- .../cluster_simulator/lib/aurora_utils.cpp | 88 ++- test/tap/tap/Makefile | 6 +- test/tap/tests/Makefile | 6 - test/tap/tests/aurora.cpp | 507 ------------------ 6 files changed, 123 insertions(+), 526 deletions(-) delete mode 100644 test/tap/tests/aurora.cpp diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index c435577530..c88ef45595 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -427,6 +427,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int cols; int affected_rows; bool run_query=true; +#ifdef TEST_AURORA + bool aws_aurora_replica_transaction=false; +#endif SQLite3_result *resultset=NULL; char *strA=NULL; char *strB=NULL; @@ -980,6 +983,18 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p run_query=false; } else { SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite3 *db = sqlite_sess->sessdb->get_db(); + if ((*proxy_sqlite3_get_autocommit)(db)==1) { + if (!sqlite_sess->sessdb->execute("BEGIN IMMEDIATE")) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to start the replica probe transaction"); + run_query=false; + } else { + aws_aurora_replica_transaction=true; + } + } + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; const int backend_port = sess->client_myds->proxy_addr.port; const std::string predicate { @@ -994,15 +1009,17 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p "SELECT replica_set_id,replica_table_present,error_code,error_msg " "FROM AWS_AURORA_REPLICA_CONTROL WHERE " + predicate }; - sqlite_sess->sessdb->execute_statement( - control_query.c_str(), &control_error, &control_cols, - &control_affected_rows, &control_result); + if (run_query) { + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + } std::string replica_set_id {}; bool replica_table_present=false; unsigned int configured_error=0; std::string configured_error_msg {}; - if (control_error == nullptr && control_result && + if (run_query && control_error == nullptr && control_result && control_result->rows_count == 1) { SQLite3_row *row=control_result->rows.front(); replica_set_id=row->fields[0] ? row->fields[0] : ""; @@ -1023,7 +1040,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p "," + (sess->client_myds->encrypted ? "1" : "0") + ")" }; - if (control_error != nullptr) { + if (!run_query) { + // The transaction-start error was already sent above. + } else if (control_error != nullptr) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, control_error); free(control_error); @@ -1175,6 +1194,12 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } #endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD if (!run_query) { +#ifdef TEST_AURORA + if (aws_aurora_replica_transaction) { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite_sess->sessdb->execute("COMMIT"); + } +#endif l_free(pkt->size-sizeof(mysql_hdr),query_no_space); l_free(query_length,query); return; @@ -1188,6 +1213,11 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); +#ifdef TEST_AURORA + if (aws_aurora_replica_transaction) { + sqlite_sess->sessdb->execute("COMMIT"); + } +#endif #if defined(TEST_GALERA) || defined(TEST_GROUPREP) if (strncasecmp("SELECT",query_no_space,6)==0) { #ifdef TEST_GALERA diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index cced5822a4..530742fc67 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1719,7 +1719,7 @@ std::pair simulate_aws_aurora_cluster_state( // change the values for the cluster state as a last step prep_aurora_init_state_err = prepare_aurora_cluster_state( - proxysql_sqlite, aurora_init_servers_state, hostgroup_attributes.empty() ? 1 : 2 + proxysql_sqlite, aurora_init_servers_state, 2 ); if (prep_aurora_init_state_err.first) { diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index 68ea1d4c86..fa63104373 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -372,6 +373,30 @@ string aurora_sql_quote(const string& value) { return quoted; } +string aurora_utc_timestamp() { + time_t now = time(nullptr); + struct tm utc_time {}; + gmtime_r(&now, &utc_time); + char timestamp[20] {}; + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &utc_time); + return timestamp; +} + +std::pair aurora_scalar_uint64(MYSQL* connection, const string& query) { + if (mysql_query(connection, query.c_str()) != 0) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_RES* result = mysql_store_result(connection); + if (result == nullptr) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_ROW row = mysql_fetch_row(result); + const bool valid = row != nullptr && row[0] != nullptr; + const uint64_t value = valid ? strtoull(row[0], nullptr, 10) : 0; + mysql_free_result(result); + return { valid ? EXIT_SUCCESS : EXIT_FAILURE, value }; +} + std::pair load_aurora_backend_addresses( std::unordered_map& addresses) { @@ -451,12 +476,44 @@ std::pair prepare_aurora_cluster_state( return error; }; - if (cleanup) { + string replica_set_list {}; + for (const auto& replica_set : replica_sets) { + if (!replica_set_list.empty()) replica_set_list += ","; + replica_set_list += aurora_sql_quote(replica_set.first); + } + + auto [checkpoint_rc, probe_checkpoint] = aurora_scalar_uint64( + proxysql_sqlite, + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (checkpoint_rc != EXIT_SUCCESS) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + } + + if (cleanup > 1) { auto [control_rc, control_error] = execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else if (cleanup == 1) { + const string delete_controls { + replica_set_list.empty() + ? "DELETE FROM AWS_AURORA_REPLICA_CONTROL" + : "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id NOT IN (" + + replica_set_list + ")" + }; + auto [control_rc, control_error] = execute_or_rollback(delete_controls); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + if (!replica_set_list.empty()) { + auto [reset_rc, reset_error] = execute_or_rollback( + "UPDATE AWS_AURORA_REPLICA_CONTROL SET replica_table_present=1," + "error_code=0,error_msg='' WHERE replica_set_id IN (" + + replica_set_list + ")"); + if (reset_rc != EXIT_SUCCESS) return { reset_rc, reset_error }; + } + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; } else { for (const auto& replica_set : replica_sets) { const string set_literal { aurora_sql_quote(replica_set.first) }; @@ -469,7 +526,7 @@ std::pair prepare_aurora_cluster_state( } } - const string timestamp { get_fmt_time() }; + const string timestamp { aurora_utc_timestamp() }; for (const auto& replica_set : replica_sets) { const string& replica_set_id = replica_set.first; std::map, bool> mapped_backends {}; @@ -525,8 +582,31 @@ std::pair prepare_aurora_cluster_state( return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } - // Allow an already scheduled monitor probe to observe the committed snapshot. - usleep(1000 * 1000); + if (cleanup == 1 && !replica_sets.empty()) { + const uint64_t deadline = monotonic_time() + 10000000; + const string observed_sets_query { + "SELECT COUNT(DISTINCT replica_set_id) FROM AWS_AURORA_REPLICA_PROBE_LOG " + "WHERE sequence_id>" + std::to_string(probe_checkpoint) + + " AND probe_kind='ordinary' AND replica_set_id IN (" + + replica_set_list + ")" + }; + do { + auto [observed_rc, observed_sets] = + aurora_scalar_uint64(proxysql_sqlite, observed_sets_query); + if (observed_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, "Unable to read the Aurora replica probe log" }; + } + if (observed_sets == replica_sets.size()) { + return { EXIT_SUCCESS, "" }; + } + usleep(50000); + } while (monotonic_time() < deadline); + + return { + EXIT_FAILURE, + "Timed out waiting for every Aurora replica set to be probed" + }; + } return { EXIT_SUCCESS, "" }; } diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index 4ceb4b1ca5..568de7a136 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -83,13 +83,13 @@ mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h +bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h tap.h command_line.h $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h +aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h $(CXX) -fPIC -c aurora_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index fedf75dea6..48912cd41c 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -227,9 +227,6 @@ tests-php: $(patsubst %,php-%,$(wildcard *-t.php)) tests-py: $(patsubst %,py-%,$(wildcard *-t.py)) tests-sh: $(patsubst %,sh-%,$(wildcard *-t.sh)) testgalera: galera_1_timeout_count galera_2_timeout_no_count -testaurora: aurora - - ### test deps targets #build_test_deps: $(TAP_LDIR)/libtap$(SHLIB_EXT) $(TEST_MARIADB_LDIR)/libmariadbclient.a $(TEST_MYSQL_LDIR)/libmysqlclient.a @@ -275,9 +272,6 @@ galera_2_timeout_no_count: galera_2_timeout_no_count.cpp $(TAP_LDIR)/libtap$(SHL generate_set_session_csv: generate_set_session_csv.cpp $(CXX) $< $(OPT) -o $@ -aurora: aurora.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) - $(CXX) -DTEST_AURORA $< ../tap/SQLite3_Server.cpp $(IDIRS) $(LDIRS) $(OPT) $(OBJ) $(MYLIBSJEMALLOC) $(MYLIBS) $(STATIC_LIBS) -o $@ - test_tokenizer-t: test_tokenizer-t.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) $(CXX) $< $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) -o $@ diff --git a/test/tap/tests/aurora.cpp b/test/tap/tests/aurora.cpp deleted file mode 100644 index 97a7b0dd0d..0000000000 --- a/test/tap/tests/aurora.cpp +++ /dev/null @@ -1,507 +0,0 @@ -#include // std::cout -#include // std::sort -#include // std::vector -#include "re2/re2.h" -#include "re2/regexp.h" -#include "proxysql.h" -#include "cpp.h" - -#include "MySQL_Logger.hpp" -#include "MySQL_Data_Stream.h" -#include "query_processor.h" -#include "SQLite3_Server.h" -#include "proxysql_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "tap.h" - -#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" -#define SELECT_VERSION_COMMENT_LEN 32 -#define SELECT_DB_USER "select DATABASE(), USER() limit 1" -#define SELECT_DB_USER_LEN 33 -#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" -#define SELECT_CHARSET_VARIOUS_LEN 115 - -#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" -#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" - -extern SQLite3_Server *GloSQLite3Server; - -static int random_replication_lag_seconds() { - static thread_local std::random_device random_source; - static thread_local std::uniform_int_distribution distribution(10, 39); - return distribution(random_source); -} - -void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { - if(!s.empty()) - s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes - for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; - for (unsigned int i=11; iexecute("DELETE FROM REPLICA_HOST_STATUS"); - sqlite3_stmt *statement=NULL; - //sqlite3 *mydb3=sessdb->get_db(); - int rc; - char *query=(char *)"INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5)"; - //rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); - auto prepared_statement = sessdb->prepare_v2(query); - rc = prepared_statement.first; - stmt_unique_ptr statement_unique = std::move(prepared_statement.second); - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - //if (rand() % 200 == 0) { - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; isize-sizeof(mysql_hdr); - query=(char *)l_alloc(query_length); - memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); - query[query_length-1]=0; - - if (sess->client_myds->proxy_addr.addr == NULL) { - struct sockaddr addr; - socklen_t addr_len=sizeof(struct sockaddr); - memset(&addr,0,addr_len); - int rc; - rc=getsockname(sess->client_myds->fd, &addr, &addr_len); - if (rc==0) { - char buf[512]; - switch (addr.sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; - inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; - inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - default: - sess->client_myds->proxy_addr.addr = strdup("unknown"); - break; - } - } else { - sess->client_myds->proxy_addr.addr = strdup("unknown"); - } - } - - char *query_no_space=(char *)l_alloc(query_length); - memcpy(query_no_space,query,query_length); - - unsigned int query_no_space_length=remove_spaces(query_no_space); - - // fix bug #925 - while (query_no_space[query_no_space_length-1]==';' || query_no_space[query_no_space_length-1]==' ') { - query_no_space_length--; - query_no_space[query_no_space_length]=0; - } - - // fix bug #1047 - if ( -/* - (!strncasecmp("BEGIN", query_no_space, strlen("BEGIN"))) - || - (!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION"))) - || - (!strncasecmp("COMMIT", query_no_space, strlen("COMMIT"))) - || - (!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK"))) - || -*/ - (!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results"))) - || - (!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL"))) - || - (!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES"))) - || - (!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT"))) - || - (!strncasecmp("/*!40100 SET @@SQL_MODE='' */", query_no_space, strlen("/*!40100 SET @@SQL_MODE='' */"))) - || - (!strncasecmp("/*!40103 SET TIME_ZONE=", query_no_space, strlen("/*!40103 SET TIME_ZONE="))) - || - (!strncasecmp("/*!80000 SET SESSION", query_no_space, strlen("/*!80000 SET SESSION"))) - || - (!strncasecmp("SET SESSION", query_no_space, strlen("SET SESSION"))) - || - (!strncasecmp("SET wait_timeout", query_no_space, strlen("SET wait_timeout"))) - ) { - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite3 *db = sqlite_sess->sessdb->get_db(); - uint16_t status=2; // autocommit - if (sqlite3_get_autocommit(db)==0) { - status = 3; // autocommit + transaction - } - GloSQLite3Server->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0, status); - run_query=false; - goto __run_query; - } - - if (query_no_space_length==17) { - if (!strncasecmp((char *)"START TRANSACTION", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==5) { - if (!strncasecmp((char *)"BEGIN", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { - if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* proxy_addr = sess->client_myds->proxy_addr.addr; - const std::string query_text = - std::string("SELECT '(ProxySQL Automated Test Server) - ") - + (proxy_addr ? proxy_addr : "") + "'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_DB_USER_LEN) { - if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* username = sess->client_myds->myconn->userinfo->username; - const std::string query_text = - std::string("SELECT \"admin\" AS 'DATABASE()', \"") - + (username ? username : "") + "\" AS 'USER()'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { - if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { - l_free(query_length,query); - char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; - query=l_strdup(query1); - query_length=strlen(query1)+1; - goto __run_query; - } - } - - if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS '@@version'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'version()'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (strncasecmp("SHOW ", query_no_space, 5)) { - goto __end_show_commands; // in the next block there are only SHOW commands - } - - if (query_no_space_length==strlen("SHOW TABLES") && !strncasecmp("SHOW TABLES",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT name AS tables FROM sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"); - query_length=strlen(query)+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; - strBl=strlen(strB); - int l=strBl+strAl-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,strA); - b[l]=0; - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES LIKE ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM sqlite_master WHERE type='table' AND name LIKE '%s'"; - strBl=strlen(strB); - char *tn=NULL; // tablename - tn=(char *)malloc(strlen(strA)); - unsigned int i=0, j=0; - while (i=3 && tbh[0]=='`' && tbh[tbh_len-1]=='`') { // tablename is quoted - size_t db_len = tbh_len - 2; - const std::string unquoted_table(tbh + 1, db_len); - free(tbh); - tbh = l_strdup(unquoted_table.c_str()); - if (!tbh) { - free(dbh); - l_free(query_length, query); - return; - } - } - const std::string table_query = - std::string("SELECT name AS 'table' , REPLACE(REPLACE(sql,' , ', X'2C0A20202020'),") - + "'CREATE TABLE " + tbh + " (','CREATE TABLE " + tbh - + " ('||X'0A20202020') AS 'Create Table' FROM " + dbh - + ".sqlite_master WHERE type='table' AND name='" + tbh + "'"; - char *buff = l_strdup(table_query.c_str()); - if (!buff) { - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length, query); - return; - } - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length,query); - query=buff; - query_length=table_query.size()+1; - goto __run_query; - } - - if ( - (query_no_space_length==strlen("SHOW DATABASES") && !strncasecmp("SHOW DATABASES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW SCHEMAS") && !strncasecmp("SHOW SCHEMAS",query_no_space, query_no_space_length)) - ) { - l_free(query_length,query); - query=l_strdup("PRAGMA DATABASE_LIST"); - query_length=strlen(query)+1; - goto __run_query; - } - -__end_show_commands: - - if (query_no_space_length==strlen("SELECT DATABASE()") && !strncasecmp("SELECT DATABASE()",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - // see issue #1022 - if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - if (sess->session_type == PROXYSQL_SESSION_SQLITE) { // no admin - if ( - (strncasecmp("PRAGMA",query_no_space,6)==0) - || - (strncasecmp("ATTACH",query_no_space,6)==0) - ) { - proxy_error("[WARNING]: Commands executed from stats interface in Admin Module: \"%s\"\n", query_no_space); - GloSQLite3Server->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Command not allowed"); - run_query=false; - } - } - -__run_query: - if (run_query) { - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - GloSQLite3Server->populate_aws_aurora_table(sess); - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - l_free(0, query); - const std::string formatted_query = cstr_format( - "SELECT %d as Seconds_Behind_Master", random_replication_lag_seconds() - ).str; - query = l_strdup(formatted_query.c_str()); - } - } - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - if (rand() % 10 == 0) { - // randomly add some latency on 10% of the traffic - sleep(2); - } - } - } - sqlite3 *db = sqlite_sess->sessdb->get_db(); - bool in_trans = false; - if (sqlite3_get_autocommit(db)==0) { - in_trans = true; - } - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans); - delete resultset; - } - l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here - l_free(query_length,query); -} From 25716bb041a7a2a6641d6874ea80900cd9a106ff Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 06:31:18 +0000 Subject: [PATCH 21/26] docs: clarify Aurora BGD status publisher contract --- include/MySQL_HostGroups_Manager.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index a340e6fc10..2dbc0e199c 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1127,8 +1127,9 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { /** * @brief Publish the node-local Aurora BGD state for one runtime row. * - * Invalid status strings and writer hostgroups not present at runtime are - * ignored. Configuration reloads do not write this column. + * The caller owns validation of the state vocabulary. Writer hostgroups not + * present at runtime are ignored. Configuration reloads do not write this + * column. */ void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** From 49c1c2995f58c4c3381b39bcd5d0e9e98c5841ec Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 08:57:33 +0000 Subject: [PATCH 22/26] refactor: clarify Aurora simulator publication modes --- ...ra-bgd-cluster-simulator-testing-design.md | 6 +- .../cluster_simulator/cluster_simulator.cpp | 15 ++-- .../cluster_simulator/lib/aurora_utils.cpp | 89 ++++--------------- .../deps/cluster_simulator/lib/aurora_utils.h | 13 +-- test/tap/groups/cluster_sim_aurora/add-hosts | 2 +- 5 files changed, 36 insertions(+), 89 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 95e01ddba6..5acaa5f887 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -205,13 +205,17 @@ Every intercepted production `REPLICA_HOST_STATUS` query records: sequence_id INTEGER PRIMARY KEY AUTOINCREMENT backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL + CHECK (probe_kind IN ('ordinary', 'bgd_membership')) replica_set_id TEXT NULL encrypted INTEGER NOT NULL CHECK (encrypted IN (0, 1)) ``` The accepted address is logged even when no control mapping exists or the query -returns an error. `replica_set_id` is NULL when the backend has no mapping. +returns an error. `probe_kind` distinguishes the ordinary production probe from +the Aurora BGD target-membership probe. `replica_set_id` is NULL when the +backend has no mapping. ### 4.4 Query handling diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index 530742fc67..dc23952a0a 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1719,7 +1719,9 @@ std::pair simulate_aws_aurora_cluster_state( // change the values for the cluster state as a last step prep_aurora_init_state_err = prepare_aurora_cluster_state( - proxysql_sqlite, aurora_init_servers_state, 2 + proxysql_sqlite, + aurora_init_servers_state, + aurora_publication_mode::reset_scenario ); if (prep_aurora_init_state_err.first) { @@ -1794,14 +1796,13 @@ std::pair simulate_aws_aurora_cluster_state( aurora_servers_state_diff(aurora_init_servers_state, aurora_new_servers_state) }; - // set the new servers state - const auto& aurora_new_state_to_set { - aurora_update_cluster_state(aurora_init_servers_state, aurora_new_servers_state) - }; // Replace the published replica sets and their complete backend mappings so // payloads can simulate members being removed (for example, autopurge coverage). - prep_aurora_final_state_err = - prepare_aurora_cluster_state(proxysql_sqlite, aurora_new_servers_state, 1); + prep_aurora_final_state_err = prepare_aurora_cluster_state( + proxysql_sqlite, + aurora_new_servers_state, + aurora_publication_mode::replace_snapshot_retaining_backends + ); if (prep_aurora_final_state_err.first) { result = internal_error(prep_aurora_final_state_err.second, __FILE__, __LINE__); diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index fa63104373..fb7b1ef9ec 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -448,7 +448,7 @@ std::pair load_aurora_backend_addresses( std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const vector& servers, - uint32_t cleanup + aurora_publication_mode mode ) { std::unordered_map backend_addresses {}; auto [host_file_rc, host_file_error] = @@ -482,21 +482,25 @@ std::pair prepare_aurora_cluster_state( replica_set_list += aurora_sql_quote(replica_set.first); } - auto [checkpoint_rc, probe_checkpoint] = aurora_scalar_uint64( - proxysql_sqlite, - "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); - if (checkpoint_rc != EXIT_SUCCESS) { - (void)mysql_query(proxysql_sqlite, "ROLLBACK"); - return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + uint64_t probe_checkpoint = 0; + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { + auto [checkpoint_rc, checkpoint] = aurora_scalar_uint64( + proxysql_sqlite, + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (checkpoint_rc != EXIT_SUCCESS) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + } + probe_checkpoint = checkpoint; } - if (cleanup > 1) { + if (mode == aurora_publication_mode::reset_scenario) { auto [control_rc, control_error] = execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; - } else if (cleanup == 1) { + } else if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { const string delete_controls { replica_set_list.empty() ? "DELETE FROM AWS_AURORA_REPLICA_CONTROL" @@ -582,7 +586,8 @@ std::pair prepare_aurora_cluster_state( return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } - if (cleanup == 1 && !replica_sets.empty()) { + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends && + !replica_sets.empty()) { const uint64_t deadline = monotonic_time() + 10000000; const string observed_sets_query { "SELECT COUNT(DISTINCT replica_set_id) FROM AWS_AURORA_REPLICA_PROBE_LOG " @@ -719,67 +724,3 @@ cluster_state_changes aurora_servers_state_diff( return result; } - -aurora_server_state_t aurora_update_state( - const aurora_server_state_t& st1, - const aurora_server_state_t& st2 -) { - aurora_server_state_t result {}; - - // SERVER_ID and DOMAIN_NAME **can't** be changed, because the are part of the server 'id'. Only the other - // fields are allowed to change, otherwise, the verification step should have failed. - - const string st1_session_id { std::get(st1) }; - const string st2_session_id { std::get(st2) }; - - int32_t st1_read_only { std::get(st1) }; - int32_t st2_read_only { std::get(st2) }; - - // Since empty 'SESSION_IDs' have no meaning, we ignore them for updated states - if (st2_session_id != "" && st1_session_id != st2_session_id) { - std::get<2>(result) = st2_session_id; - } - - if (st2_read_only != -1 && st1_read_only != st2_read_only) { - std::get<3>(result) = st2_read_only; - } - - return result; -} - -vector aurora_update_cluster_state( - const vector& servers_state_p, - const vector& servers_state_n -) { - vector result {}; - - vector s_servers_state_p { sort_aurora_server_state(servers_state_p) }; - vector s_servers_state_n { sort_aurora_server_state(servers_state_n) }; - - // find the differences - for (const auto& server_state_n : s_servers_state_n) { - for (const auto& server_state_p : s_servers_state_p) { - const string n_server_state_id { - std::get(server_state_n) + ":" + - std::get(server_state_n) - }; - const string p_server_state_id { - std::get(server_state_p) + ":" + - std::get(server_state_p) - }; - - bool diff_server_status = - ( n_server_state_id == p_server_state_id ) && - ( server_state_n != server_state_p ); - - if (diff_server_status) { - const aurora_server_state_t server_state_update { - aurora_update_state(server_state_p, server_state_n) - }; - result.push_back(server_state_update); - } - } - } - - return result; -} diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index f8ba8dd2e3..5006d50ccd 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -50,6 +50,12 @@ enum class aurora_state_id { new_state = 1 }; +enum class aurora_publication_mode { + replace_sets, + replace_snapshot_retaining_backends, + reset_scenario +}; + std::pair extract_aurora_servers_state( const aurora_state_id& state_id, const json& aurora_test_def, @@ -69,7 +75,7 @@ std::pair prepare_mysql_aurora_hostgroups( std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const std::vector& servers, - uint32_t cleanup = 0 + aurora_publication_mode mode = aurora_publication_mode::replace_sets ); /** @@ -93,9 +99,4 @@ cluster_state_changes aurora_servers_state_diff( const std::vector& servers_state_n ); -std::vector aurora_update_cluster_state( - const std::vector& servers_state_p, - const std::vector& servers_state_n -); - #endif diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 94b8d811c7..128b632715 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -2,7 +2,7 @@ # Format: " " per line; '#' comments allowed. # These are injected into the ProxySQL container's /etc/hosts via Docker # --add-host by test/infra/control/start-proxysql-isolated.bash when -# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/aurora-sim/env.sh). +# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/cluster_sim_aurora/env.sh). # Cluster 1 host.1.11.aws-test.com 127.0.1.11 From e437b3b1ac3f42f00f9604001c55fb21f10b4a81 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 17:51:17 +0000 Subject: [PATCH 23/26] feat: add Aurora BGD discovery and membership tracking --- include/MySQL_Monitor.hpp | 94 ++++ lib/MySQL_Monitor.cpp | 522 ++++++++++++++++++ test/tap/groups/cluster_sim_aurora/add-hosts | 5 + test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 290 ++++++++++ .../tap/tests/test_aurora_bgd_discovery-t.cpp | 278 ++++++++++ 6 files changed, 1190 insertions(+) create mode 100644 test/tap/tap/aurora_bgd_tap.h create mode 100644 test/tap/tests/test_aurora_bgd_discovery-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 38c379d825..163d05106f 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -564,6 +564,85 @@ struct AWS_RDS_BGD_Probe_Host { int use_ssl = 0; }; +/** + * @brief Aurora blue/green deployment phase published by the Aurora worker. + */ +enum class AWS_Aurora_BGD_Status { + NONE = 0, + AVAILABLE, + SWITCHOVER_INITIATED, + SWITCHOVER_IN_PROGRESS, + SWITCHOVER_IN_POST_PROCESSING, + SWITCHOVER_COMPLETED, +}; + +/** + * @brief Stable identity of one Aurora blue/green deployment. + */ +struct AWS_Aurora_BGD_Fingerprint { + std::string target_id; + std::string target_endpoint; + int target_port = 0; + + bool empty() const { + return target_id.empty() || target_endpoint.empty() || target_port <= 0; + } + + bool operator==(const AWS_Aurora_BGD_Fingerprint& rhs) const { + return target_id == rhs.target_id + && target_endpoint == rhs.target_endpoint + && target_port == rhs.target_port; + } +}; + +/** + * @brief One production or target Aurora member retained by the BGD worker. + */ +struct AWS_Aurora_BGD_Member { + std::string server_id; + std::string normalized_server_id; + std::string session_id; + std::string hostname; + std::string production_hostname; + std::string target_ip; + int port = 0; + int use_ssl = 0; + bool is_writer = false; + bool traffic_pin_applied = false; +}; + +/** + * @brief State carried by one existing per-writer Aurora monitor worker. + */ +struct AWS_Aurora_BGD_State { + unsigned int writer_hg = 0; + unsigned int reader_hg = 0; + int green_writer_hg = -1; + int green_reader_hg = -1; + unsigned int check_interval_ms = 0; + unsigned int check_timeout_ms = 0; + int target_use_ssl = 0; + std::string domain_name; + + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + AWS_Aurora_BGD_Fingerprint fingerprint; + std::vector production_probe_hosts; + std::vector production_members; + std::vector target_members; + + bool production_snapshot_frozen = false; + bool production_probe_suspended = false; + bool target_snapshot_complete = false; + + bool has_complete_target_snapshot() const { + return target_snapshot_complete; + } +}; + +// Maps an Aurora switchover status enum to its runtime string. +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status); + /** * @brief Switchover state carried by RDS BGD worker thread. * @@ -723,6 +802,21 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief Refresh the Aurora BGD worker's last complete production snapshot. + * + * @details Invalid or incomplete observations retain the previous snapshot. + */ + void aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); + /** + * @brief Run the topology and target-membership probes owned by an Aurora worker. + * + * @details Discovery is serialized with the ordinary Aurora probe. This method + * validates topology before publishing status and replaces target membership + * only with a complete, unambiguous, fully resolved snapshot. + */ + void aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st); /** * @brief AWS RDS BGD monitor thread entry point. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a76bb3b2f5..53ec72e02a 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6057,6 +6057,504 @@ static void shuffle_hosts(host_def_t *array, size_t n) { } } +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { + switch (status) { + case AWS_Aurora_BGD_Status::NONE: + return "NONE"; + case AWS_Aurora_BGD_Status::AVAILABLE: + return BGD_STATUS_AVAILABLE; + case AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED: + return BGD_STATUS_INITIATED; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS: + return BGD_STATUS_IN_PROGRESS; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING: + return BGD_STATUS_POST_PROC; + case AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED: + return BGD_STATUS_COMPLETED; + } + return "NONE"; +} + +namespace { + +struct AWS_Aurora_BGD_Query_Result { + unique_ptr mmsd; + int rc = 1; + unsigned int mysql_error = 0; +}; + +struct AWS_Aurora_BGD_Topology_Observation { + bool valid = false; + bool completed = false; + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + AWS_Aurora_BGD_Fingerprint fingerprint; +}; + +static std::string aws_aurora_bgd_normalize_server_id(const std::string& server_id) { + const size_t green_pos = server_id.find("-green-"); + return green_pos == std::string::npos ? server_id : server_id.substr(0, green_pos); +} + +static std::string aws_aurora_bgd_member_hostname( + const std::string& server_id, const std::string& domain_name +) { + if (domain_name.empty()) { + return server_id; + } + return domain_name.front() == '.' + ? server_id + domain_name + : server_id + "." + domain_name; +} + +static bool aws_aurora_bgd_status_from_raw( + const std::string& raw_status, AWS_Aurora_BGD_Status& status +) { + if (strcasecmp(raw_status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + status = AWS_Aurora_BGD_Status::AVAILABLE; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_INITIATED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_POST_PROC) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_COMPLETED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + } else { + return false; + } + return true; +} + +static bool aws_aurora_bgd_required_topology_fields(const AWS_RDS_Topology_Node& node) { + return !node.id.empty() && !node.endpoint.empty() && node.port > 0 + && !node.role.empty() && !node.status.empty(); +} + +static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( + const AWS_RDS_Topology_Result& topology +) { + AWS_Aurora_BGD_Topology_Observation observation; + if (!topology.blue_green || topology.nodes.empty()) { + return observation; + } + + const AWS_RDS_Topology_Node* source = nullptr; + const AWS_RDS_Topology_Node* target = nullptr; + for (const AWS_RDS_Topology_Node& node : topology.nodes) { + if (!aws_aurora_bgd_required_topology_fields(node)) { + return observation; + } + if (strcasecmp(node.role.c_str(), BGD_ROLE_SOURCE) == 0) { + if (source != nullptr) { + return observation; + } + source = &node; + } else if (strcasecmp(node.role.c_str(), BGD_ROLE_TARGET) == 0) { + if (target != nullptr) { + return observation; + } + target = &node; + } else { + return observation; + } + } + + if (target == nullptr || !aws_aurora_bgd_status_from_raw(target->status, observation.status)) { + return observation; + } + + if (observation.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + if (topology.nodes.size() != 1 || source != nullptr) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + observation.completed = true; + } else { + if (topology.nodes.size() != 2 || source == nullptr + || strcasecmp(source->status.c_str(), target->status.c_str()) != 0) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + } + + observation.fingerprint = { + target->id, + target->endpoint, + target->port, + }; + observation.valid = !observation.fingerprint.empty(); + return observation; +} + +static bool aws_aurora_bgd_select_reachable_host( + const std::vector& hosts, + unsigned int writer_hg, + AWS_RDS_BGD_Probe_Host& selected +) { + if (hosts.empty()) { + return false; + } + + const size_t first = static_cast(rand()) % hosts.size(); + for (size_t offset = 0; offset < hosts.size(); ++offset) { + const AWS_RDS_BGD_Probe_Host& host = hosts[(first + offset) % hosts.size()]; + if (GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port)) { + selected = host; + return true; + } + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, writer_hg, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + } + return false; +} + +static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( + const AWS_RDS_BGD_Probe_Host& host, + unsigned int writer_hg, + unsigned int timeout_ms, + MySQL_Monitor_State_Data_Task_Type task_type, + const char* query +) { + AWS_Aurora_BGD_Query_Result out; + out.mmsd.reset(new MySQL_Monitor_State_Data( + task_type, const_cast(host.hostname.c_str()), host.port, host.use_ssl, writer_hg)); + MySQL_Monitor_State_Data* mmsd = out.mmsd.get(); + mmsd->writer_hostgroup = writer_hg; + mmsd->aws_aurora_check_timeout_ms = timeout_ms; + mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); + + bool new_connection = false; + if (mmsd->mysql == nullptr) { + new_connection = true; + if (!mmsd->create_new_connection()) { + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + return out; + } + GloMyMon->My_Conn_Pool->conn_register(mmsd); + } + + std::atomic_bool worker_stop {false}; + out.rc = GloMyMon->aws_rds_bgd_async_query(mmsd, query, worker_stop); + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + + if (out.rc != 0) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else if (new_connection) { + if (mmsd->set_wait_timeout()) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } + } else { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } + + return out; +} + +static void aws_aurora_bgd_set_status( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status status +) { + if (st.status == status) { + return; + } + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_aurora_bgd_status_str(st.status), aws_aurora_bgd_status_str(status)); + st.status = status; + MyHGM->update_aws_aurora_bgd_status(st.writer_hg, aws_aurora_bgd_status_str(status)); +} + +static bool aws_aurora_bgd_same_production_snapshot( + const std::vector& lhs, + const std::vector& rhs +) { + if (lhs.size() != rhs.size()) { + return false; + } + for (const AWS_Aurora_BGD_Member& member : lhs) { + auto found = std::find_if(rhs.begin(), rhs.end(), [&](const AWS_Aurora_BGD_Member& candidate) { + return candidate.normalized_server_id == member.normalized_server_id + && candidate.is_writer == member.is_writer; + }); + if (found == rhs.end()) { + return false; + } + } + return true; +} + +static bool aws_aurora_bgd_parse_target_membership( + AWS_Aurora_BGD_State& st, MYSQL_RES* result, + std::vector& snapshot +) { + if (result == nullptr || st.production_members.empty()) { + return false; + } + + const unsigned int num_fields = mysql_num_fields(result); + MYSQL_FIELD* fields = mysql_fetch_fields(result); + int server_id_idx = -1; + int session_id_idx = -1; + int is_current_idx = -1; + for (unsigned int i = 0; i < num_fields; ++i) { + if (fields[i].name == nullptr) { + continue; + } + if (strcasecmp(fields[i].name, "SERVER_ID") == 0) { + server_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "SESSION_ID") == 0) { + session_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "IS_CURRENT") == 0) { + is_current_idx = static_cast(i); + } + } + if (server_id_idx < 0 || session_id_idx < 0 || is_current_idx < 0) { + return false; + } + + std::unordered_map production_by_id; + for (const AWS_Aurora_BGD_Member& member : st.production_members) { + if (!production_by_id.emplace(member.normalized_server_id, &member).second) { + return false; + } + } + + std::unordered_map previous_reader_session_by_id; + std::unordered_map previous_reader_id_by_session; + std::unordered_map previous_ip_by_id; + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + previous_ip_by_id[member.normalized_server_id] = member.target_ip; + if (!member.is_writer) { + previous_reader_session_by_id[member.normalized_server_id] = member.session_id; + previous_reader_id_by_session[member.session_id] = member.normalized_server_id; + } + } + + std::unordered_set target_ids; + std::unordered_set reader_sessions; + unsigned int writers = 0; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + if (row[is_current_idx] == nullptr) { + continue; + } + const bool is_current = atoi(row[is_current_idx]) != 0 + || strcasecmp(row[is_current_idx], "true") == 0; + if (!is_current) { + continue; + } + if (row[server_id_idx] == nullptr || row[server_id_idx][0] == '\0' + || row[session_id_idx] == nullptr || row[session_id_idx][0] == '\0') { + return false; + } + + AWS_Aurora_BGD_Member member; + member.server_id = row[server_id_idx]; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row[session_id_idx]; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !target_ids.insert(member.normalized_server_id).second) { + return false; + } + + auto production = production_by_id.find(member.normalized_server_id); + if (production == production_by_id.end() + || production->second->is_writer != member.is_writer) { + return false; + } + member.production_hostname = production->second->hostname; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.port = st.fingerprint.target_port; + member.use_ssl = st.target_use_ssl; + + if (member.is_writer) { + writers++; + } else { + if (!reader_sessions.insert(member.session_id).second) { + return false; + } + auto old_session = previous_reader_session_by_id.find(member.normalized_server_id); + if (old_session != previous_reader_session_by_id.end() + && old_session->second != member.session_id) { + return false; + } + auto old_id = previous_reader_id_by_session.find(member.session_id); + if (old_id != previous_reader_id_by_session.end() + && old_id->second != member.normalized_server_id) { + return false; + } + } + + size_t ip_count = 0; + std::string resolved_ip = MySQL_Monitor::dns_lookup(member.hostname, false, &ip_count); + if (resolved_ip.empty()) { + const int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + std::vector ips = dns_resolve(member.hostname, ai_family); + if (!ips.empty()) { + resolved_ip = ips.front(); + } + } + if (resolved_ip.empty()) { + return false; + } + auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); + member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() + ? previous_ip->second : resolved_ip; + snapshot.push_back(std::move(member)); + } + + return writers == 1 && snapshot.size() == st.production_members.size(); +} + +static std::vector aws_aurora_bgd_target_probe_hosts( + const AWS_Aurora_BGD_State& st +) { + std::vector hosts; + if (st.has_complete_target_snapshot()) { + hosts.reserve(st.target_members.size()); + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + hosts.push_back({member.hostname, member.port, member.use_ssl}); + } + } else if (!st.fingerprint.empty()) { + hosts.push_back({ + st.fingerprint.target_endpoint, + st.fingerprint.target_port, + st.target_use_ssl, + }); + } + return hosts; +} + +} // namespace + +void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result +) { + if (st.production_snapshot_frozen || result.error != nullptr + || result.host_statuses == nullptr || result.host_statuses->empty()) { + return; + } + + std::vector snapshot; + std::unordered_set member_ids; + unsigned int writers = 0; + for (const AWS_Aurora_replica_host_status_entry* row : *result.host_statuses) { + if (row == nullptr || row->server_id == nullptr || row->server_id[0] == '\0' + || row->session_id == nullptr) { + return; + } + AWS_Aurora_BGD_Member member; + member.server_id = row->server_id; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row->session_id; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.production_hostname = member.hostname; + member.port = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().port; + member.use_ssl = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().use_ssl; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !member_ids.insert(member.normalized_server_id).second) { + return; + } + writers += member.is_writer ? 1 : 0; + snapshot.push_back(std::move(member)); + } + if (writers != 1) { + return; + } + + if (!aws_aurora_bgd_same_production_snapshot(st.production_members, snapshot)) { + st.target_snapshot_complete = false; + } + st.production_members = std::move(snapshot); +} + +void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { + const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; + const bool discovery_admitted = configured_green_hgs + || st.status != AWS_Aurora_BGD_Status::NONE + || mysql_thread___aws_blue_green_deployment_auto_discovery != 0; + if (!discovery_admitted) { + return; + } + + std::vector topology_hosts = st.has_complete_target_snapshot() + ? aws_aurora_bgd_target_probe_hosts(st) + : st.production_probe_hosts; + AWS_RDS_BGD_Probe_Host topology_host; + if (!aws_aurora_bgd_select_reachable_host(topology_hosts, st.writer_hg, topology_host)) { + proxy_error("No node is pingable for AWS Aurora BGD topology checks with writer HG %u\n", st.writer_hg); + return; + } + + if (st.topology_state == TOPOLOGY_TABLE_CHECK) { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); + if (query.rc == 0 && query.mmsd->result + && mysql_num_rows(query.mmsd->result) > 0) { + st.topology_state = TOPOLOGY_METADATA_FETCH; + } + } else { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + if (query.rc != 0) { + if (query.mysql_error == 1146) { + st.topology_state = TOPOLOGY_TABLE_CHECK; + } + } else if (query.mmsd->result && mysql_num_rows(query.mmsd->result) > 0) { + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); + AWS_Aurora_BGD_Topology_Observation observation = + aws_aurora_bgd_validate_topology(topology); + if (observation.valid && !observation.completed) { + const bool same_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint; + const bool rollback_transition = + static_cast(observation.status) < static_cast(st.status); + if (same_deployment && !rollback_transition) { + st.fingerprint = observation.fingerprint; + st.target_use_ssl = topology_host.use_ssl; + if (observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED) { + st.production_snapshot_frozen = true; + } + aws_aurora_bgd_set_status(st, observation.status); + } + } + } + } + + if (st.status == AWS_Aurora_BGD_Status::NONE || st.fingerprint.empty()) { + return; + } + + std::vector membership_hosts = aws_aurora_bgd_target_probe_hosts(st); + AWS_RDS_BGD_Probe_Host membership_host; + if (!aws_aurora_bgd_select_reachable_host(membership_hosts, st.writer_hg, membership_host)) { + proxy_error("No node is pingable for AWS Aurora BGD membership checks with writer HG %u\n", st.writer_hg); + return; + } + + AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( + membership_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS); + if (membership.rc != 0) { + return; + } + + std::vector snapshot; + if (aws_aurora_bgd_parse_target_membership(st, membership.mmsd->result, snapshot)) { + st.target_members = std::move(snapshot); + st.target_snapshot_complete = true; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS Aurora BGD [wHG=%u rHG=%u]: retained complete target membership with %zu members\n", + st.writer_hg, st.reader_hg, st.target_members.size()); + } +} + void * monitor_AWS_Aurora_thread_HG(void *arg) { unsigned int wHG = *(unsigned int *)arg; unsigned int rHG = 0; @@ -6071,6 +6569,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { unsigned int autopurge_missing_checks = 0; std::string domain_name; std::map autopurge_counter; + AWS_Aurora_BGD_State bgd_state; + bgd_state.writer_hg = wHG; set_thread_name("MonitorAuroraHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS Aurora writer HG %u\n", wHG); @@ -6118,6 +6618,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (domain_name.empty() && r->fields[12]) { domain_name = r->fields[12]; } + if (r->fields[13] && r->fields[13][0]) { + bgd_state.green_writer_hg = atoi(r->fields[13]); + } + if (r->fields[14] && r->fields[14][0]) { + bgd_state.green_reader_hg = atoi(r->fields[14]); + } } } host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*num_hosts); @@ -6127,9 +6633,21 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { hpa[cur_host_idx].host = strdup(r->fields[2]); hpa[cur_host_idx].port = atoi(r->fields[3]); hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); + bgd_state.production_probe_hosts.push_back({ + r->fields[2], + atoi(r->fields[3]), + atoi(r->fields[4]), + }); cur_host_idx++; } } + bgd_state.reader_hg = rHG; + bgd_state.check_interval_ms = check_interval_ms; + bgd_state.check_timeout_ms = check_timeout_ms; + bgd_state.domain_name = domain_name; + if (!bgd_state.production_probe_hosts.empty()) { + bgd_state.target_use_ssl = bgd_state.production_probe_hosts.front().use_ssl; + } // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access // can table place later when accessing 'hpa[cur_host_idx]'. if (cur_host_idx >= num_hosts) { @@ -6388,6 +6906,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mysql_free_result(mmsd->result); mmsd->result=NULL; } + GloMyMon->aws_aurora_bgd_refresh_production_snapshot(bgd_state, *ase); if (lasts_ase[ase_idx]) { AWS_Aurora_status_entry * l_ase = lasts_ase[ase_idx]; @@ -6477,6 +6996,9 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } + if (GloMyMon->shutdown == false && exit_now == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + } } __exit_monitor_AWS_Aurora_thread_HG_now: if (mmsd) { diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 128b632715..cb6b8a8746 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -53,6 +53,9 @@ aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com 127.0.11.20 aurora-a-green-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 aurora-a-green-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 +aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com 127.0.11.23 # Canonical names observed after target-member rename retain target addresses. aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 @@ -65,5 +68,7 @@ aurora-b-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.12 aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com 127.0.12.20 aurora-b-green-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-green-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-reader-1-green-r8t4.b1.us-east-1.rds.amazonaws.com 127.0.12.22 aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 1c53e22d9b..631ac58cec 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -363,6 +363,7 @@ "test_client_limit_error-t" : [ "todo-g1" ], "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h new file mode 100644 index 0000000000..09e12ff4b3 --- /dev/null +++ b/test/tap/tap/aurora_bgd_tap.h @@ -0,0 +1,290 @@ +#ifndef TAP_TESTS_AURORA_BGD_TAP_H +#define TAP_TESTS_AURORA_BGD_TAP_H + +#include +#include +#include +#include +#include +#include + +#include "aurora_bgd_simulator.h" +#include "tap.h" + +using namespace std; + +struct Aurora_BGD_Test_Deployment { + string name; + string domain_name; + string blue_replica_set; + string target_replica_set; + Aurora_BGD_Endpoint target_cluster_endpoint; + Aurora_BGD_Membership_Set production; + Aurora_BGD_Membership_Set target; + string source_topology_id; + string target_topology_id; +}; + +inline Aurora_BGD_Member aurora_bgd_member( + string server_id, string session_id, Aurora_BGD_Endpoint endpoint, bool current = true +) { + Aurora_BGD_Member member; + member.server_id = move(server_id); + member.session_id = move(session_id); + member.endpoint = move(endpoint); + member.last_update_timestamp = "2099-01-01 00:00:00"; + member.is_current = current; + return member; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_a() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment A"; + deployment.domain_name = ".a1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-a"; + deployment.target_replica_set = "aurora-bgd-target-a"; + deployment.source_topology_id = "aurora-bgd-source-a"; + deployment.target_topology_id = "aurora-bgd-target-a"; + deployment.target_cluster_endpoint = { + "aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com", "127.0.11.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-a-writer", "MASTER_SESSION_ID", + {"aurora-a-writer.a1.us-east-1.rds.amazonaws.com", "127.0.11.11", 3306}), + aurora_bgd_member("aurora-a-reader-1", "reader-a-1", + {"aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com", "127.0.11.12", 3306}), + aurora_bgd_member("aurora-a-reader-2", "reader-a-2", + {"aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com", "127.0.11.13", 3306}), + }, + {} + }; + for (Aurora_BGD_Member& member : deployment.production.members) { + deployment.production.serving_endpoints.push_back(member.endpoint); + } + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-a-writer-green-k7m2", "MASTER_SESSION_ID", + {"aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com", "127.0.11.21", 3306}), + aurora_bgd_member("aurora-a-reader-1-green-p4q8", "reader-a-1", + {"aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com", "127.0.11.22", 3306}), + aurora_bgd_member("aurora-a-reader-2-green-v9n3", "reader-a-2", + {"aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com", "127.0.11.23", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + for (Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + return deployment; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment B writer-only"; + deployment.domain_name = ".b1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-b"; + deployment.target_replica_set = "aurora-bgd-target-b"; + deployment.source_topology_id = "aurora-bgd-source-b"; + deployment.target_topology_id = "aurora-bgd-target-b"; + deployment.target_cluster_endpoint = { + "aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com", "127.0.12.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-b-writer", "MASTER_SESSION_ID", + {"aurora-b-writer.b1.us-east-1.rds.amazonaws.com", "127.0.12.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-b-writer-green-h2s6", "MASTER_SESSION_ID", + {"aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com", "127.0.12.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + +inline string aurora_bgd_sql_quote(const string& value) { + string quoted {"'"}; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +inline int aurora_bgd_execute_all(MYSQL* admin, const vector& queries) { + for (const string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag("Aurora BGD admin query failed: %s; query: %s", mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +inline int aurora_bgd_admin_cleanup(MYSQL* admin) { + return aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "DELETE FROM mysql_aws_aurora_hostgroups", + "DELETE FROM mysql_servers", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +inline int aurora_bgd_admin_setup( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, + bool auto_discovery +) { + vector queries { + "SET mysql-monitor_username='aurora1'", + "SET mysql-monitor_password='pass1'", + "SET mysql-monitor_enabled='true'", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + + string(auto_discovery ? "true" : "false") + "'", + }; + + string green_columns = green_writer_hg >= 0 + ? to_string(green_writer_hg) + "," + to_string(green_reader_hg) + : "NULL,NULL"; + queries.push_back( + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment) VALUES (" + + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + + ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + + ",200,100,800,0,1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); + + for (Aurora_BGD_Member& member : deployment.production.members) { + const int hostgroup = member.session_id == "MASTER_SESSION_ID" ? writer_hg : reader_hg; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(member.endpoint.hostname) + + ",3306,'ONLINE',0,'Aurora BGD production member')"); + } + queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +inline vector aurora_bgd_available_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.source_topology_id, + deployment.production.members.front().endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + "AVAILABLE", + }, + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "AVAILABLE", + }, + }; +} + +inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { + vector backends = deployment.production.backends(); + vector target_backends = deployment.target.backends(); + backends.insert(backends.end(), target_backends.begin(), target_backends.end()); + return backends; +} + +inline int aurora_bgd_publish( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment +) { + if (sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + if (sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_available_topology(deployment)); +} + +inline int aurora_bgd_wait_for_status( + MYSQL* admin, int writer_hg, const string& status, uint32_t timeout_seconds +) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND bgd_status=" + aurora_bgd_sql_quote(status); + return wait_for_cond(admin, query, timeout_seconds); +} + +inline rc_t aurora_bgd_wait_for_topology_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + BGD_Probe_Kind kind, uint32_t timeout_ms +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const BGD_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +inline rc_t aurora_bgd_wait_for_replica_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + Aurora_Replica_Probe_Kind kind, uint32_t timeout_ms, const string& replica_set = "" +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind + && (replica_set.empty() || log.replica_set_id == replica_set)) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +#endif // TAP_TESTS_AURORA_BGD_TAP_H diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp new file mode 100644 index 0000000000..6462d1c141 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_discovery-t.cpp @@ -0,0 +1,278 @@ +/** + * @file test_aurora_bgd_discovery-t.cpp + * @brief Aurora BGD AVAILABLE discovery, three probes, and fail-closed mapping. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_status_is(MYSQL* admin, int writer_hg, const string& status) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == status; +} + +bool runtime_production_unchanged(MYSQL* admin, int writer_hg, int reader_hg, size_t members) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id IN (" + + to_string(writer_hg) + "," + to_string(reader_hg) + ")"); + return rc == EXIT_SUCCESS && rows.size() == members; +} + +bool retain_complete_target_after_membership( + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows +) { + auto [seq_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, rows, deployment.target.backends()) != EXIT_SUCCESS) { + return false; + } + + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + auto [first_rc, first_probe] = aurora_bgd_wait_for_replica_probe( + sim, sequence, target_members, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [next_rc, next_probe] = aurora_bgd_wait_for_replica_probe( + sim, first_probe.sequence_id, target_members, + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + return next_rc == EXIT_SUCCESS; +} + +bool invalid_topology_retains_available( + MYSQL* admin, + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows, + int writer_hg +) { + auto [seq_rc, sequence] = sim.probe_log_last_sequence(); + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + if (seq_rc != EXIT_SUCCESS || sim.topology_update( + aurora_bgd_topology_backends(deployment), rows) != EXIT_SUCCESS) { + return false; + } + auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, target_members, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return probe_rc == EXIT_SUCCESS && runtime_status_is(admin, writer_hg, "AVAILABLE"); +} + +int main() { + plan(18); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1510; + const int reader_hg = 1511; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, deployment, writer_hg, reader_hg, 1512, 1513, false) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure deployment A"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "valid SOURCE/TARGET topology publishes AVAILABLE"); + + { + auto [rc, logs] = sim.replica_probe_log_since(0); + bool ordinary_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.replica_set_id == deployment.blue_replica_set) { + ordinary_seen = true; + } + } + ok(rc == EXIT_SUCCESS && ordinary_seen, + "the existing worker continues ordinary Aurora probing in AVAILABLE"); + } + + vector blue_backends = deployment.production.backends(); + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "topology discovery starts on a reachable production member"); + } + { + auto [rc, probe] = sim.wait_for_replica_probe_log( + 0, deployment.target_cluster_endpoint.backend(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + 0, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "TARGET cluster endpoint bootstraps target membership"); + } + + vector target_backends; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_backends.push_back(member.endpoint.backend()); + } + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "complete membership moves topology probes to target members"); + } + { + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "complete membership moves membership probes to target members"); + } + + vector complete_rows = deployment.target.replica_rows(); + vector incomplete_rows {complete_rows[0], complete_rows[1]}; + ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), + "an incomplete target result retains the previous complete target selection"); + + vector duplicate_writer_rows = complete_rows; + duplicate_writer_rows[1].session_id = "MASTER_SESSION_ID"; + ok(retain_complete_target_after_membership(sim, deployment, duplicate_writer_rows), + "multiple current target writers retain the previous complete snapshot"); + + vector unresolved_rows = complete_rows; + unresolved_rows[2].server_id = "aurora-a-reader-2-green-missing"; + ok(retain_complete_target_after_membership(sim, deployment, unresolved_rows), + "an unresolved target member retains the previous complete snapshot"); + + vector mismatched_session_rows = complete_rows; + mismatched_session_rows[1].session_id = "reader-a-1-replaced"; + ok(retain_complete_target_after_membership(sim, deployment, mismatched_session_rows), + "a reader SESSION_ID mismatch retains the previous complete snapshot"); + + vector stale_extra_rows = complete_rows; + Aurora_Replica_Row stale_writer = complete_rows.front(); + stale_writer.server_id = "unrelated-stale-writer"; + stale_writer.is_current = false; + stale_extra_rows.push_back(stale_writer); + ok(retain_complete_target_after_membership(sim, deployment, stale_extra_rows), + "IS_CURRENT=0 rows are excluded from the complete target snapshot"); + + if (sim.replica_update( + deployment.target_replica_set, complete_rows, deployment.target.backends()) != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + + vector valid_topology = aurora_bgd_available_topology(deployment); + vector missing_identity = valid_topology; + missing_identity[1].id.clear(); + ok(invalid_topology_retains_available(admin, sim, deployment, missing_identity, writer_hg), + "missing TARGET identity does not replace AVAILABLE"); + + vector source_only {valid_topology.front()}; + ok(invalid_topology_retains_available(admin, sim, deployment, source_only, writer_hg), + "SOURCE-only topology does not replace AVAILABLE"); + + vector mismatched_status = valid_topology; + mismatched_status[0].status = "SWITCHOVER_INITIATED"; + ok(invalid_topology_retains_available(admin, sim, deployment, mismatched_status, writer_hg), + "mismatched SOURCE/TARGET statuses do not replace AVAILABLE"); + + vector unknown_status = valid_topology; + unknown_status[0].status = "UNSUPPORTED_STATUS"; + unknown_status[1].status = "UNSUPPORTED_STATUS"; + ok(invalid_topology_retains_available(admin, sim, deployment, unknown_status, writer_hg), + "unsupported topology status is not copied into runtime state"); + + ok(runtime_production_unchanged(admin, writer_hg, reader_hg, deployment.production.members.size()), + "AVAILABLE discovery performs no production routing action"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset state before writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + + Aurora_BGD_Test_Deployment writer_only = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, writer_only) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, writer_only, 1520, 1521, -1, -1, true) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1520, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "auto-discovery admits a row without configured green hostgroups"); + { + vector writer_target { + writer_only.target.members.front().endpoint.backend() + }; + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, writer_only.target_replica_set); + ok(rc == EXIT_SUCCESS, "a writer-only production cluster accepts a writer-only target snapshot"); + } + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD discovery state"); + return EXIT_FAILURE; + } + return exit_status(); +} From d5f486a7a6569b8a9a6daa80c85ff84a115c5fdf Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 18:30:39 +0000 Subject: [PATCH 24/26] feat: handle Aurora BGD active switchover --- include/MySQL_Monitor.hpp | 9 + lib/MySQL_Monitor.cpp | 101 +++- test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 22 +- test/tap/tests/test_aurora_bgd_active-t.cpp | 480 ++++++++++++++++++++ 5 files changed, 604 insertions(+), 9 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_active-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 163d05106f..b4fb0798f5 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -809,6 +809,15 @@ class MySQL_Monitor { */ void aws_aurora_bgd_refresh_production_snapshot( AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); + /** + * @brief Apply the routing actions for an accepted active Aurora BGD state. + * + * @details INITIATED only suspends production probing. IN_PROGRESS demotes + * the snapshotted writer on entry. POST_PROCESSING applies each complete + * member mapping once and restores canonical writer placement. + */ + void aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed); /** * @brief Run the topology and target-membership probes owned by an Aurora worker. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 53ec72e02a..094066ef64 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6326,8 +6326,10 @@ static bool aws_aurora_bgd_parse_target_membership( std::unordered_map previous_reader_session_by_id; std::unordered_map previous_reader_id_by_session; std::unordered_map previous_ip_by_id; + std::unordered_map previous_action_by_id; for (const AWS_Aurora_BGD_Member& member : st.target_members) { previous_ip_by_id[member.normalized_server_id] = member.target_ip; + previous_action_by_id[member.normalized_server_id] = member.traffic_pin_applied; if (!member.is_writer) { previous_reader_session_by_id[member.normalized_server_id] = member.session_id; previous_reader_id_by_session[member.session_id] = member.normalized_server_id; @@ -6404,6 +6406,9 @@ static bool aws_aurora_bgd_parse_target_membership( auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() ? previous_ip->second : resolved_ip; + auto previous_action = previous_action_by_id.find(member.normalized_server_id); + member.traffic_pin_applied = previous_action != previous_action_by_id.end() + && previous_action->second; snapshot.push_back(std::move(member)); } @@ -6472,6 +6477,71 @@ void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( st.production_members = std::move(snapshot); } +void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed +) { + if (st.status < AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + || st.status > AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + return; + } + + st.production_snapshot_frozen = true; + st.production_probe_suspended = true; + + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS + && status_changed) { + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer == st.production_members.end()) { + proxy_error( + "AWS Aurora BGD [wHG=%u rHG=%u]: cannot demote writer without a complete production snapshot\n", + st.writer_hg, st.reader_hg); + return; + } + MyHGM->update_aws_aurora_set_reader( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + return; + } + + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING + || !st.has_complete_target_snapshot()) { + return; + } + + bool applied_member_action = false; + bool writer_pin_applied = false; + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + dns_cache->pin(member.production_hostname, member.target_ip); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = true; + applied_member_action = true; + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: repointed production '%s' to target IP %s\n", + st.writer_hg, st.reader_hg, member.production_hostname.c_str(), + member.target_ip.c_str()); + } + writer_pin_applied |= member.is_writer && member.traffic_pin_applied; + } + + if (!writer_pin_applied || (!status_changed && !applied_member_action)) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + } +} + void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs @@ -6516,12 +6586,11 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) const bool rollback_transition = static_cast(observation.status) < static_cast(st.status); if (same_deployment && !rollback_transition) { + const bool status_changed = st.status != observation.status; st.fingerprint = observation.fingerprint; st.target_use_ssl = topology_host.use_ssl; - if (observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED) { - st.production_snapshot_frozen = true; - } aws_aurora_bgd_set_status(st, observation.status); + aws_aurora_bgd_apply_active_actions(st, status_changed); } } } @@ -6552,6 +6621,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) proxy_debug(PROXY_DEBUG_MONITOR, 7, "AWS Aurora BGD [wHG=%u rHG=%u]: retained complete target membership with %zu members\n", st.writer_hg, st.reader_hg, st.target_members.size()); + aws_aurora_bgd_apply_active_actions(st, false); } } @@ -6713,6 +6783,17 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); continue; } + if (bgd_state.production_probe_suspended) { + if (mmsd) { + delete mmsd; + mmsd = NULL; + } + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + continue; + } //proxy_info("Running check AWS Aurora writer HG %u\n", wHG); found_pingable_host = false; @@ -6759,7 +6840,10 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); - next_loop_at = t1 + check_interval_ms * 1000; + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } #ifdef TEST_AURORA @@ -6996,9 +7080,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } - if (GloMyMon->shutdown == false && exit_now == false) { - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); - } + if (GloMyMon->shutdown == false && exit_now == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + } } __exit_monitor_AWS_Aurora_thread_HG_now: if (mmsd) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 631ac58cec..b06d7f0ad2 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -364,6 +364,7 @@ "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index 09e12ff4b3..cfc7aa78fd 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -141,14 +141,17 @@ inline int aurora_bgd_admin_cleanup(MYSQL* admin) { "LOAD MYSQL VARIABLES TO RUNTIME", "DELETE FROM mysql_aws_aurora_hostgroups", "DELETE FROM mysql_servers", + "UPDATE mysql_users SET default_hostgroup=0 WHERE username='testuser'", "LOAD MYSQL SERVERS TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", }); } inline int aurora_bgd_admin_setup( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, - bool auto_discovery + bool auto_discovery, int check_interval_ms = 100, + bool writer_is_also_reader = false ) { vector queries { "SET mysql-monitor_username='aurora1'", @@ -156,6 +159,8 @@ inline int aurora_bgd_admin_setup( "SET mysql-monitor_enabled='true'", "SET mysql-aws_blue_green_deployment_auto_discovery='" + string(auto_discovery ? "true" : "false") + "'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(writer_hg) + + " WHERE username='testuser'", }; string green_columns = green_writer_hg >= 0 @@ -169,7 +174,9 @@ inline int aurora_bgd_admin_setup( "autopurge_missing_checks,comment) VALUES (" + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + - ",200,100,800,0,1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); + ",200," + to_string(check_interval_ms) + ",800," + + to_string(writer_is_also_reader ? 1 : 0) + + ",1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); for (Aurora_BGD_Member& member : deployment.production.members) { const int hostgroup = member.session_id == "MASTER_SESSION_ID" ? writer_hg : reader_hg; @@ -180,6 +187,7 @@ inline int aurora_bgd_admin_setup( } queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + queries.push_back("LOAD MYSQL USERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } @@ -204,6 +212,16 @@ inline vector aurora_bgd_available_topology( }; } +inline vector aurora_bgd_topology( + Aurora_BGD_Test_Deployment& deployment, const string& status +) { + vector rows = aurora_bgd_available_topology(deployment); + for (BGD_Topology_Row& row : rows) { + row.status = status; + } + return rows; +} + inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { vector backends = deployment.production.backends(); vector target_backends = deployment.target.backends(); diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp new file mode 100644 index 0000000000..cbcc256222 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -0,0 +1,480 @@ +/** + * @file test_aurora_bgd_active-t.cpp + * @brief Aurora BGD active-state probe suspension and routing behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + if (aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("Error: failed to configure the routing test user"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int user_rc = admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds); +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +bool fast_membership_without_ordinary( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms, uint64_t minimum_membership_probes +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + uint64_t membership_probes = 0; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set) { + membership_probes++; + } + } + return membership_probes >= minimum_membership_probes; +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs +) { + vector queries; + for (size_t i = 0; i < deployment.production.members.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", rc, mysql_error(client)); + } + mysql_close(client); + if (rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + const Aurora_Replica_Probe_Log* routed_probe = nullptr; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + routed_probe = &log; + } + } + if (routed_probe == nullptr + || routed_probe->backend.host != expected_backend.host + || routed_probe->backend.port != expected_backend.port) { + diag( + "Ordinary Aurora query reached %s:%d; expected %s:%d", + routed_probe ? routed_probe->backend.host.c_str() : "", + routed_probe ? routed_probe->backend.port : 0, + expected_backend.host.c_str(), expected_backend.port); + return false; + } + return true; +} + +bool route_members_to_expected_ips( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected_backend = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected_backend)) { + return false; + } + } + return true; +} + +int64_t member_route_pool_count(MYSQL* admin, const vector& route_hgs) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup IN (" + hostgroups + ")"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_member_route_pool_count( + MYSQL* admin, const vector& route_hgs, const string& comparison +) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup IN (" + hostgroups + ")", + kWaitSeconds); +} + +int main() { + plan(20); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1530; + const int reader_hg = 1531; + const vector route_hgs {1534, 1535, 1536}; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, writer_hg, reader_hg, 1532, 1533, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, deployment, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure the active switchover scenario"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "active scenario starts from a complete AVAILABLE snapshot"); + + if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_INITIATED"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "INITIATED is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "INITIATED does not change writer placement"); + + auto [active_seq_rc, active_sequence] = sim.replica_probe_log_last_sequence(); + ok(active_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, active_sequence, deployment.target_replica_set, 650, 3), + "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); + + vector competing_source = deployment.production.replica_rows(); + competing_source[0].session_id = "source-observed-reader"; + competing_source[1].session_id = "MASTER_SESSION_ID"; + auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); + if (source_change_rc != EXIT_SUCCESS || sim.replica_update( + deployment.blue_replica_set, competing_source, deployment.production.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to publish the competing source role observation"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, source_change_sequence, deployment.target_replica_set, 350, 2), + "changed source roles cannot compete while production probing is suspended"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "IN_PROGRESS is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "IN_PROGRESS demotes the snapshotted production writer"); + ok(runtime_server_count( + admin, writer_hg, deployment.production.members[1].endpoint.hostname, 0) + && runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), + "the competing source observation does not promote a reader"); + + auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); + if (repeat_progress_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) + && wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "repeated IN_PROGRESS retains the one demotion and active probe policy"); + + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, false) + && member_route_pool_count(admin, route_hgs) + >= static_cast(route_hgs.size()), + "all member routes use source IPs and hold pre-cutover pools"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "POST_PROCESSING is published without a target-writability gate"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "POST_PROCESSING restores the canonical writer placement"); + ok(runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + && runtime_server_count( + admin, reader_hg, deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + "POST_PROCESSING leaves canonical readers ONLINE and eligible"); + ok(wait_for_member_route_pool_count(admin, route_hgs, "=0") == EXIT_SUCCESS, + "POST_PROCESSING retires the pre-cutover member pools"); + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, true), + "POST_PROCESSING pins every production hostname to its cached target IP"); + + const int64_t target_pool_count = member_route_pool_count(admin, route_hgs); + auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); + if (target_pool_count < static_cast(route_hgs.size()) + || repeat_post_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to prepare repeated POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + vector target_backends = deployment.target.backends(); + auto [repeat_probe_rc, repeat_probe] = aurora_bgd_wait_for_topology_probe( + sim, repeat_post_sequence, target_backends, BGD_Probe_Kind::metadata, + kProbeTimeoutMs); + ok(repeat_probe_rc == EXIT_SUCCESS + && member_route_pool_count(admin, route_hgs) >= target_pool_count, + "repeated POST_PROCESSING does not replay completed member retirement"); + + auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, post_sequence, deployment.target_replica_set, 350, 2), + "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset before the incomplete-snapshot scenario"); + cleanup(admin, sim); + return exit_status(); + } + + const int gated_writer_hg = 1540; + const int gated_reader_hg = 1541; + const int gated_route_hg = 1544; + Aurora_BGD_Test_Deployment gated = aurora_bgd_deployment_b_writer_only(); + vector invalid_target = gated.target.replica_rows(); + invalid_target.front().server_id = "unpaired-target-writer"; + if (sim.replica_update( + gated.blue_replica_set, gated.production.replica_rows(), gated.production.backends()) + != EXIT_SUCCESS + || sim.replica_update( + gated.target_replica_set, invalid_target, gated.target.backends()) + != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(gated), aurora_bgd_available_topology(gated)) + != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, gated, gated_writer_hg, gated_reader_hg, 1542, 1543, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, gated, {gated_route_hg}) != EXIT_SUCCESS) { + diag("Error: failed to configure incomplete target membership"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, gated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "an incomplete target snapshot can publish AVAILABLE without routing"); + if (publish_status(sim, gated, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, gated_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: incomplete scenario did not publish POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + const int gated_default_rc = set_default_hostgroup(admin, gated_route_hg); + ok(gated_default_rc == EXIT_SUCCESS && route_to_expected_backend( + cl, sim, gated.production.members.front().endpoint.backend()), + "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); + + auto [gated_refresh_seq_rc, gated_refresh_sequence] = sim.replica_probe_log_last_sequence(); + if (gated_refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( + gated.target_replica_set, gated.target.replica_rows(), gated.target.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + auto [gated_probe_rc, gated_probe] = aurora_bgd_wait_for_replica_probe( + sim, gated_refresh_sequence, gated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, gated.target_replica_set); + bool gated_target_routing = false; + if (gated_probe_rc == EXIT_SUCCESS + && wait_for_member_route_pool_count(admin, {gated_route_hg}, "=0") == EXIT_SUCCESS + && set_default_hostgroup(admin, gated_route_hg) == EXIT_SUCCESS) { + gated_target_routing = route_to_expected_backend( + cl, sim, gated.target.members.front().endpoint.backend()); + } + ok(gated_target_routing, + "repeated POST_PROCESSING applies routing after membership becomes complete"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD active-state test data"); + return EXIT_FAILURE; + } + return exit_status(); +} From 931df1e4d43d8194e61732da1a1d6288add460b3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 18:58:41 +0000 Subject: [PATCH 25/26] feat: complete Aurora BGD switchover --- include/MySQL_Monitor.hpp | 14 + lib/MySQL_Monitor.cpp | 144 ++++- test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 14 + .../tests/test_aurora_bgd_completion-t.cpp | 549 ++++++++++++++++++ 5 files changed, 702 insertions(+), 20 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_completion-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index b4fb0798f5..6727c6db0d 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -818,6 +818,20 @@ class MySQL_Monitor { */ void aws_aurora_bgd_apply_active_actions( AWS_Aurora_BGD_State& st, bool status_changed); + /** + * @brief Run immediate effect-driven cleanup for TARGET completion. + * + * @details Reconciles writer placement, removes mapped traffic pins, drains + * eligible configured green pools, resumes production probing, and enters + * the completed latch while retaining the deployment fingerprint. + */ + void aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint); + /** + * @brief Release the completed latch after a successful topology drain. + */ + void aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st); /** * @brief Run the topology and target-membership probes owned by an Aurora worker. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 094066ef64..1504b1cdf7 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6542,6 +6542,91 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( } } +void MySQL_Monitor::aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint +) { + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + && st.fingerprint == completed_fingerprint) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + } + + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + dns_cache->remove(member.production_hostname); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + } + + struct green_server_t { + int hostgroup; + std::string hostname; + int port; + }; + std::vector green_servers; + MyHGM->wrlock(); + for (int hostgroup : {st.green_writer_hg, st.green_reader_hg}) { + if (hostgroup < 0) { + continue; + } + MyHGC* hgc = MyHGM->MyHGC_find(hostgroup); + if (hgc == nullptr || hgc->mysrvs == nullptr) { + continue; + } + for (unsigned int i = 0; i < hgc->mysrvs->cnt(); ++i) { + MySrvC* server = hgc->mysrvs->idx(i); + if (server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + green_servers.push_back({hostgroup, server->address, server->port}); + MyHGM->drain_server_connections(server->address, server->port); + } + } + MyHGM->wrunlock(); + + for (const green_server_t& server : green_servers) { + dns_cache->remove(server.hostname); + My_Conn_Pool->purge_connections(server.hostname.c_str(), server.port); + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, server.hostgroup, + server.hostname.c_str(), server.port); + } + + st.fingerprint = completed_fingerprint; + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover cleanup complete; completion latched\n", + st.writer_hg, st.reader_hg); +} + +void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + return; + } + + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + st.target_members.clear(); + st.target_snapshot_complete = false; + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::NONE); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: topology drained; completion latch released\n", + st.writer_hg, st.reader_hg); +} + void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs @@ -6564,9 +6649,12 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); - if (query.rc == 0 && query.mmsd->result - && mysql_num_rows(query.mmsd->result) > 0) { - st.topology_state = TOPOLOGY_METADATA_FETCH; + if (query.rc == 0 && query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) > 0) { + st.topology_state = TOPOLOGY_METADATA_FETCH; + } else { + aws_aurora_bgd_release_completed_latch(st); + } } } else { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( @@ -6576,27 +6664,40 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (query.mysql_error == 1146) { st.topology_state = TOPOLOGY_TABLE_CHECK; } - } else if (query.mmsd->result && mysql_num_rows(query.mmsd->result) > 0) { - AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); - AWS_Aurora_BGD_Topology_Observation observation = - aws_aurora_bgd_validate_topology(topology); - if (observation.valid && !observation.completed) { - const bool same_deployment = st.fingerprint.empty() - || st.fingerprint == observation.fingerprint; - const bool rollback_transition = - static_cast(observation.status) < static_cast(st.status); - if (same_deployment && !rollback_transition) { - const bool status_changed = st.status != observation.status; - st.fingerprint = observation.fingerprint; - st.target_use_ssl = topology_host.use_ssl; - aws_aurora_bgd_set_status(st, observation.status); - aws_aurora_bgd_apply_active_actions(st, status_changed); + } else if (query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) == 0) { + aws_aurora_bgd_release_completed_latch(st); + } else { + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); + AWS_Aurora_BGD_Topology_Observation observation = + aws_aurora_bgd_validate_topology(topology); + if (observation.valid && observation.completed) { + const bool same_active_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + if (same_active_deployment) { + aws_aurora_bgd_apply_completion(st, observation.fingerprint); + } + } else if (observation.valid) { + const bool same_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint; + const bool rollback_transition = + static_cast(observation.status) < static_cast(st.status); + if (same_deployment && !rollback_transition) { + const bool status_changed = st.status != observation.status; + st.fingerprint = observation.fingerprint; + st.target_use_ssl = topology_host.use_ssl; + aws_aurora_bgd_set_status(st, observation.status); + aws_aurora_bgd_apply_active_actions(st, status_changed); + } } } } } - if (st.status == AWS_Aurora_BGD_Status::NONE || st.fingerprint.empty()) { + if (st.status == AWS_Aurora_BGD_Status::NONE + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + || st.fingerprint.empty()) { return; } @@ -7154,7 +7255,6 @@ void * MySQL_Monitor::monitor_aws_aurora() { pthread_mutex_unlock(&aws_aurora_mutex); if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS Aurora monitoring\n"); - last_raw_checksum = new_raw_checksum; if (pthreads_array) { // wait all threads to terminate for (unsigned int i=0; i < hgs_num; i++) { @@ -7168,6 +7268,10 @@ void * MySQL_Monitor::monitor_aws_aurora() { } hgs_num = 0; pthread_mutex_lock(&aws_aurora_mutex); + // Configuration can change while the previous workers are being joined. + // Launch replacements from the current snapshot and retain that checksum, + // so the coordinator never waits on a worker created from newer state. + last_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); // scan all the writer HGs unsigned int num_rows = AWS_Aurora_Hosts_resultset->rows_count; if (num_rows) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index b06d7f0ad2..9afa842be7 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -365,6 +365,7 @@ "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index cfc7aa78fd..a38cb4bb5f 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -222,6 +222,20 @@ inline vector aurora_bgd_topology( return rows; } +inline vector aurora_bgd_completed_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + deployment.target_cluster_endpoint.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }, + }; +} + inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { vector backends = deployment.production.backends(); vector target_backends = deployment.target.backends(); diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp new file mode 100644 index 0000000000..fdc22e1cbe --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -0,0 +1,549 @@ +/** + * @file test_aurora_bgd_completion-t.cpp + * @brief Aurora BGD completion cleanup and terminal-latch behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int publish_completed( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& serving_deployment, + Aurora_BGD_Test_Deployment& completed_deployment +) { + return sim.topology_update( + aurora_bgd_topology_backends(serving_deployment), + aurora_bgd_completed_topology(completed_deployment)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_route( + MYSQL* admin, int hostgroup, const string& hostname, const string& status = "ONLINE" +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + + ",3306," + aurora_bgd_sql_quote(status) + ",'Aurora BGD completion route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups +) { + vector queries; + for (size_t i = 0; i < hostgroups.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroups[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD completion member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int add_green_servers( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int green_writer_hg, int green_reader_hg +) { + vector queries; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + const int hostgroup = i == 0 ? green_writer_hg : green_reader_hg; + const string status = i + 1 == deployment.target.members.size() + ? "OFFLINE_SOFT" : "ONLINE"; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.target.members[i].endpoint.hostname) + + ",3306," + aurora_bgd_sql_quote(status) + + ",'Aurora BGD configured green member')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (query_rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", + mysql_errno(client), mysql_error(client)); + } + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected_backend.host + && log.backend.port == expected_backend.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups, bool target +) { + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (set_default_hostgroup(admin, hostgroups[i]) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup=" + to_string(hostgroup)); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_pool_count(MYSQL* admin, int hostgroup, const string& comparison) { + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup), + kWaitSeconds); +} + +bool server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_policy( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool writer_is_also_reader +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (writer_is_also_reader ? "1" : "0") + ")", + kWaitSeconds); +} + +int wait_for_writer_demotion( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=0) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1)", + kWaitSeconds); +} + +bool completion_probe_policy( + BGD_Simulator& sim, uint64_t sequence, + Aurora_BGD_Test_Deployment& deployment, uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool ordinary_on_production = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership) { + return false; + } + if (log.probe_kind != Aurora_Replica_Probe_Kind::ordinary) { + continue; + } + for (const Endpoint& backend : deployment.production.backends()) { + ordinary_on_production |= log.backend.host == backend.host + && log.backend.port == backend.port; + } + } + return ordinary_on_production; +} + +bool wait_for_topology_observation( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, deployment.production.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int main() { + plan(28); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + // Completion from IN_PROGRESS reconciles the writer without replaying POST_PROCESSING. + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + const int progress_writer_hg = 1550; + const int progress_reader_hg = 1551; + if (aurora_bgd_publish(sim, progress) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, progress, progress_writer_hg, progress_reader_hg, + 1552, 1553, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure completion-from-IN_PROGRESS scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, progress_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "completion-from-IN_PROGRESS scenario reaches AVAILABLE"); + if (publish_status(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to reach IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(wait_for_writer_demotion( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname) == EXIT_SUCCESS, + "IN_PROGRESS demotes the writer before completion"); + if (publish_completed(sim, progress, progress) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to enter the completed latch from IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "TARGET-only completion publishes SWITCHOVER_COMPLETED"); + ok(wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "completion restores the demoted writer without replaying POST_PROCESSING"); + auto [progress_replica_seq_rc, progress_replica_sequence] = + sim.replica_probe_log_last_sequence(); + ok(progress_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, progress_replica_sequence, progress, 750), + "completion resumes ordinary production probing and stops membership probing"); + auto [repeat_progress_seq_rc, repeat_progress_sequence] = sim.probe_log_last_sequence(); + bool repeated_progress = repeat_progress_seq_rc == EXIT_SUCCESS + && publish_completed(sim, progress, progress) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_progress_sequence, progress); + ok(repeated_progress && wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "repeated completion is a no-op while latched"); + auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); + bool error_retained = error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(progress), 1205, "simulated completion timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, error_sequence, progress); + ok(error_retained && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS, + "topology query errors retain the completed latch"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "confirmed topology absence releases the completed latch to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // Completion after POST_PROCESSING removes pins and drains configured green pools once. + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const int post_writer_hg = 1560; + const int post_reader_hg = 1561; + const int green_writer_hg = 1562; + const int green_reader_hg = 1563; + const vector route_hgs {1564, 1565, 1566}; + if (aurora_bgd_publish(sim, post) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, post, post_writer_hg, post_reader_hg, + green_writer_hg, green_reader_hg, false, 300, true) != EXIT_SUCCESS + || add_green_servers(admin, post, green_writer_hg, green_reader_hg) != EXIT_SUCCESS + || add_member_routes(admin, post, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, post_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "POST_PROCESSING completion scenario reaches AVAILABLE"); + if (publish_status(sim, post, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS + || publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance through POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "active deployment advances through POST_PROCESSING"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "POST_PROCESSING routes every production member to its target IP"); + const int64_t target_route_pool = pool_count(admin, route_hgs.front()); + bool green_pool_ready = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + ok(target_route_pool >= 1 && green_pool_ready, + "pre-completion target and configured-green pools are established"); + if (publish_completed(sim, post, post) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to complete the POST_PROCESSING scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "completion after POST_PROCESSING enters the terminal latch"); + ok(wait_for_writer_policy( + admin, post_writer_hg, post_reader_hg, + post.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "completion preserves canonical writer_is_also_reader placement"); + ok(pool_count(admin, route_hgs.front()) >= target_route_pool, + "completion does not repeat retirement of post-cutover production pools"); + ok(wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "completion drains eligible configured-green pools immediately"); + ok(server_count( + admin, green_reader_hg, + post.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), + "completion preserves configured green rows and OFFLINE status"); + const int post_completion_route_hg = 1567; + ok(add_route( + admin, post_completion_route_hg, + post.production.members.front().endpoint.hostname) == EXIT_SUCCESS + && set_default_hostgroup(admin, post_completion_route_hg) == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, post.production.members.front().endpoint.backend()), + "completion removes the production traffic pin without DNS verification"); + auto [post_replica_seq_rc, post_replica_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, post_replica_sequence, post, 750), + "the completed latch uses configured cadence and canonical production probes"); + bool recreated_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + auto [same_completed_seq_rc, same_completed_sequence] = sim.probe_log_last_sequence(); + bool same_completion_seen = same_completed_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, post) == EXIT_SUCCESS + && wait_for_topology_observation(sim, same_completed_sequence, post); + ok(recreated_green_pool && same_completion_seen && pool_count(admin, green_writer_hg) >= 1, + "repeated completion does not drain a pool created while latched"); + auto [latched_error_seq_rc, latched_error_sequence] = sim.probe_log_last_sequence(); + bool latched_error_seen = latched_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated latched timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, latched_error_sequence, post); + ok(latched_error_seen && aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS + && pool_count(admin, green_writer_hg) >= 1, + "query errors neither release the latch nor repeat completion cleanup"); + Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); + auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); + bool different_seen = different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, different_sequence, post); + ok(different_seen && wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "a different completed deployment fingerprint rearms and runs its cleanup"); + bool second_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()); + auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); + bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_different_sequence, post); + ok(second_green_pool && repeated_different && pool_count(admin, green_writer_hg) >= 1, + "the new fingerprint is retained and its repeated completion is a no-op"); + ok(sim.topology_delete(aurora_bgd_topology_backends(post)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, post_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "successful empty topology releases the rearmed completed latch"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before direct-completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // A first observation at completion has no target map and does not replay prior phases. + Aurora_BGD_Test_Deployment direct = aurora_bgd_deployment_b_writer_only(); + if (sim.replica_update( + direct.blue_replica_set, direct.production.replica_rows(), + direct.production.backends()) != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(direct), + aurora_bgd_completed_topology(direct)) != EXIT_SUCCESS) { + diag("Error: failed to publish direct completion inputs"); + cleanup(admin, sim); + return exit_status(); + } + auto [direct_seq_rc, direct_sequence] = sim.replica_probe_log_last_sequence(); + if (aurora_bgd_admin_setup( + admin, direct, 1580, 1581, -1, -1, true, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure direct completion"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1580, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "late entry directly at completion enters the terminal latch"); + ok(wait_for_writer_policy( + admin, 1580, 1581, + direct.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "direct completion leaves canonical writer placement unchanged"); + auto [direct_logs_rc, direct_logs] = sim.replica_probe_log_since(direct_sequence); + bool direct_membership_probe = false; + for (const Aurora_Replica_Probe_Log& log : direct_logs) { + direct_membership_probe |= + log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; + } + ok(direct_seq_rc == EXIT_SUCCESS && direct_logs_rc == EXIT_SUCCESS + && !direct_membership_probe, + "direct completion does not manufacture target membership or replay active phases"); + ok(sim.topology_delete(aurora_bgd_topology_backends(direct)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1580, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "direct-completion latch rearms only after topology drain"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD completion test data"); + return EXIT_FAILURE; + } + return exit_status(); +} From 561bff5bdb54c65644cdddf518be9d0d7718110f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 20:05:47 +0000 Subject: [PATCH 26/26] feat: harden Aurora BGD lifecycle handling --- include/MySQL_HostGroups_Manager.h | 4 + include/MySQL_Monitor.hpp | 67 ++- lib/MySQL_HostGroups_Manager.cpp | 30 +- lib/MySQL_Monitor.cpp | 560 +++++++++++------- test/tap/groups/cluster_sim_aurora/add-hosts | 5 + test/tap/groups/groups.json | 3 + test/tap/tap/aurora_bgd_tap.h | 32 + .../tests/test_aurora_bgd_cluster_sync-t.cpp | 306 ++++++++++ .../tap/tests/test_aurora_bgd_lifecycle-t.cpp | 397 +++++++++++++ .../tests/test_aurora_bgd_resilience-t.cpp | 412 +++++++++++++ 10 files changed, 1578 insertions(+), 238 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_lifecycle-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_resilience-t.cpp diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 3ba9955883..bd81cffc35 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1227,6 +1227,10 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { //void update_aws_aurora_set_reader(int _whid, int _rhid, char *_hostname, int _port); bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); + void update_aws_aurora_set_writer( + int _whid, int _rhid, char *server_id, bool verbose, + const char *domain_name, int aurora_port, int writer_is_also_reader, + int new_reader_weight); void update_aws_aurora_set_reader(int _whid, int _rhid, char *server_id); /** * @brief Updates the resultset and corresponding checksum used by Monitor for AWS Aurora. diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 6727c6db0d..4a2637f61a 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -576,6 +576,40 @@ enum class AWS_Aurora_BGD_Status { SWITCHOVER_COMPLETED, }; +/** + * @brief Column positions in `AWS_Aurora_Hosts_resultset`. + */ +enum AWS_Aurora_Hosts_Column { + AWS_AURORA_WRITER_HOSTGROUP = 0, + AWS_AURORA_READER_HOSTGROUP, + AWS_AURORA_HOSTNAME, + AWS_AURORA_PORT, + AWS_AURORA_USE_SSL, + AWS_AURORA_MAX_LAG_MS, + AWS_AURORA_CHECK_INTERVAL_MS, + AWS_AURORA_CHECK_TIMEOUT_MS, + AWS_AURORA_ADD_LAG_MS, + AWS_AURORA_MIN_LAG_MS, + AWS_AURORA_LAG_NUM_CHECKS, + AWS_AURORA_AUTOPURGE_MISSING_CHECKS, + AWS_AURORA_DOMAIN_NAME, + AWS_AURORA_GREEN_WRITER_HOSTGROUP, + AWS_AURORA_GREEN_READER_HOSTGROUP, + AWS_AURORA_WRITER_IS_ALSO_READER, + AWS_AURORA_NEW_READER_WEIGHT, + AWS_AURORA_HOSTS_COLUMNS, +}; + +/** + * @brief Monitor worker owned by one Aurora writer hostgroup. + */ +struct AWS_Aurora_BGD_Worker { + int writer_hg = 0; + pthread_t thread {}; + std::atomic_bool worker_stop {false}; + std::atomic current_checksum {0}; +}; + /** * @brief Stable identity of one Aurora blue/green deployment. */ @@ -619,8 +653,15 @@ struct AWS_Aurora_BGD_State { unsigned int reader_hg = 0; int green_writer_hg = -1; int green_reader_hg = -1; + unsigned int max_lag_ms = 0; unsigned int check_interval_ms = 0; unsigned int check_timeout_ms = 0; + unsigned int add_lag_ms = 0; + unsigned int min_lag_ms = 0; + unsigned int lag_num_checks = 1; + unsigned int autopurge_missing_checks = 0; + int writer_is_also_reader = 0; + int new_reader_weight = 1; int target_use_ssl = 0; std::string domain_name; @@ -802,6 +843,19 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief Load the configuration rows for one Aurora worker. + * + * @return true when the rows still match the coordinator-provided checksum. + */ + bool aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate); + /** + * @brief Refresh only configuration-derived fields of a running Aurora worker. + */ + bool aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at); /** * @brief Refresh the Aurora BGD worker's last complete production snapshot. * @@ -828,6 +882,16 @@ class MySQL_Monitor { void aws_aurora_bgd_apply_completion( AWS_Aurora_BGD_State& st, const AWS_Aurora_BGD_Fingerprint& completed_fingerprint); + /** + * @brief Reverse applied pre-completion effects and enter a safe earlier state. + * + * @details Removes only applied traffic pins, drains affected production + * pools, restores canonical writer placement, resumes ordinary probing, + * and optionally clears the deployment identity. + */ + void aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment); /** * @brief Release the completed latch after a successful topology drain. */ @@ -839,7 +903,8 @@ class MySQL_Monitor { * validates topology before publishing status and replaces target membership * only with a complete, unambiguous, fully resolved snapshot. */ - void aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st); + void aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop); /** * @brief AWS RDS BGD monitor thread entry point. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 17ca164329..fd572ef66c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7196,6 +7196,15 @@ int MySQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& ad // FIXME: complete this!! void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid, char *_server_id, bool verbose) { + update_aws_aurora_set_writer( + _whid, _rhid, _server_id, verbose, nullptr, -1, -1, -1); +} + +void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( + int _whid, int _rhid, char *_server_id, bool verbose, + const char *domain_name_override, int aurora_port_override, + int writer_is_also_reader_override, int new_reader_weight_override +) { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; @@ -7203,7 +7212,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q=NULL; char *error=NULL; //q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=offline_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; - q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; + q=(char *)"SELECT hostgroup_id FROM mysql_servers WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; int writer_is_also_reader=0; int new_reader_weight = 1; @@ -7213,7 +7222,14 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid int aurora_port = 3306; char *domain_name = strdup((char *)""); int read_HG=-1; - { + if (domain_name_override != nullptr) { + free(domain_name); + domain_name = strdup(domain_name_override); + aurora_port = aurora_port_override; + writer_is_also_reader = writer_is_also_reader_override; + new_reader_weight = new_reader_weight_override; + read_HG = _rhid; + } else { pthread_mutex_lock(&AWS_Aurora_Info_mutex); std::map::iterator it2; it2 = AWS_Aurora_Info_Map.find(_writer_hostgroup); @@ -7317,10 +7333,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q1 = NULL; char *q2 = NULL; char *error=NULL; - q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers.comment FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; - q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers_incoming.comment FROM mysql_servers_incoming JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; + q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; + q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; query = (char *)malloc(strlen(q2)+128); - sprintf(query,q1,_writer_hostgroup); + sprintf(query,q1,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7331,7 +7347,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid delete resultset_servers; resultset_servers = NULL; } - sprintf(query,q2,_writer_hostgroup); + sprintf(query,q2,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7550,7 +7566,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name," - " green_writer_hostgroup, green_reader_hostgroup FROM mysql_servers" + " green_writer_hostgroup, green_reader_hostgroup, writer_is_also_reader, new_reader_weight FROM mysql_servers" " JOIN mysql_aws_aurora_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 1504b1cdf7..d9adc334b5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6034,29 +6034,6 @@ bool AWS_Aurora_monitor_node::add_entry(AWS_Aurora_status_entry *ase) { } -typedef struct _host_def_t { - char *host; - int port; - int use_ssl; -} host_def_t; - -static void shuffle_hosts(host_def_t *array, size_t n) { - char tmp[sizeof(host_def_t)]; - char *arr = (char *)array; - size_t stride = sizeof(host_def_t) * sizeof(char); - - if (n > 1) { - size_t i; - for (i = 0; i < n - 1 ; ++i) { - size_t rnd = (size_t) fastrand(); - size_t j = i + rnd / (0x7FFF / (n - i) + 1); - memcpy(tmp, arr + j * stride, sizeof(host_def_t)); - memcpy(arr + j * stride, arr + i * stride, sizeof(host_def_t)); - memcpy(arr + i * stride, tmp, sizeof(host_def_t)); - } - } -} - const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { switch (status) { case AWS_Aurora_BGD_Status::NONE: @@ -6215,9 +6192,14 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( unsigned int writer_hg, unsigned int timeout_ms, MySQL_Monitor_State_Data_Task_Type task_type, - const char* query + const char* query, + std::atomic_bool& worker_stop ) { AWS_Aurora_BGD_Query_Result out; + if (worker_stop.load()) { + out.rc = 2; + return out; + } out.mmsd.reset(new MySQL_Monitor_State_Data( task_type, const_cast(host.hostname.c_str()), host.port, host.use_ssl, writer_hg)); MySQL_Monitor_State_Data* mmsd = out.mmsd.get(); @@ -6235,7 +6217,6 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( GloMyMon->My_Conn_Pool->conn_register(mmsd); } - std::atomic_bool worker_stop {false}; out.rc = GloMyMon->aws_rds_bgd_async_query(mmsd, query, worker_stop); out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; @@ -6538,7 +6519,9 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); if (writer != st.production_members.end()) { MyHGM->update_aws_aurora_set_writer( - st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); } } @@ -6556,7 +6539,9 @@ void MySQL_Monitor::aws_aurora_bgd_apply_completion( [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); if (writer != st.production_members.end()) { MyHGM->update_aws_aurora_set_writer( - st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); } for (const AWS_Aurora_BGD_Member& member : st.target_members) { @@ -6612,6 +6597,51 @@ void MySQL_Monitor::aws_aurora_bgd_apply_completion( st.writer_hg, st.reader_hg); } +void MySQL_Monitor::aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment +) { + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + continue; + } + dns_cache->remove(member.production_hostname); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = false; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); + } + + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + if (clear_deployment) { + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + } + + aws_aurora_bgd_set_status(st, next_status); + if (next_status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + && next_status <= AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + aws_aurora_bgd_apply_active_actions(st, true); + } + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: rollback cleanup complete; status '%s'\n", + st.writer_hg, st.reader_hg, aws_aurora_bgd_status_str(st.status)); +} + void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { return; @@ -6627,7 +6657,12 @@ void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st.writer_hg, st.reader_hg); } -void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { +void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop +) { + if (worker_stop.load()) { + return; + } const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs || st.status != AWS_Aurora_BGD_Status::NONE @@ -6648,10 +6683,17 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (st.topology_state == TOPOLOGY_TABLE_CHECK) { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker_stop); + if (worker_stop.load()) { + return; + } if (query.rc == 0 && query.mmsd->result) { if (mysql_num_rows(query.mmsd->result) > 0) { st.topology_state = TOPOLOGY_METADATA_FETCH; + } else if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); } else { aws_aurora_bgd_release_completed_latch(st); } @@ -6659,14 +6701,23 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) } else { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker_stop); + if (worker_stop.load()) { + return; + } if (query.rc != 0) { if (query.mysql_error == 1146) { st.topology_state = TOPOLOGY_TABLE_CHECK; } } else if (query.mmsd->result) { if (mysql_num_rows(query.mmsd->result) == 0) { - aws_aurora_bgd_release_completed_latch(st); + if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); + } else { + aws_aurora_bgd_release_completed_latch(st); + } } else { AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); AWS_Aurora_BGD_Topology_Observation observation = @@ -6683,7 +6734,10 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) || st.fingerprint == observation.fingerprint; const bool rollback_transition = static_cast(observation.status) < static_cast(st.status); - if (same_deployment && !rollback_transition) { + if (same_deployment && rollback_transition) { + aws_aurora_bgd_apply_rollback( + st, observation.status, false); + } else if (same_deployment) { const bool status_changed = st.status != observation.status; st.fingerprint = observation.fingerprint; st.target_use_ssl = topology_host.use_ssl; @@ -6697,7 +6751,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (st.status == AWS_Aurora_BGD_Status::NONE || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED - || st.fingerprint.empty()) { + || st.fingerprint.empty() || worker_stop.load()) { return; } @@ -6710,7 +6764,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( membership_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS); + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS, worker_stop); if (membership.rc != 0) { return; } @@ -6726,19 +6780,102 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) } } +bool MySQL_Monitor::aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate +) { + SQLite3_result cluster_result(AWS_AURORA_HOSTS_COLUMNS); + pthread_mutex_lock(&aws_aurora_mutex); + if (AWS_Aurora_Hosts_resultset) { + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + if (atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]) == writer_hg) { + cluster_result.add_row(row); + } + } + } + pthread_mutex_unlock(&aws_aurora_mutex); + + if (cluster_result.rows.empty() || cluster_result.raw_checksum() != current_checksum) { + return false; + } + + candidate.writer_hg = writer_hg; + bool first_row = true; + for (SQLite3_row* row : cluster_result.rows) { + if (first_row) { + candidate.reader_hg = atoi(row->fields[AWS_AURORA_READER_HOSTGROUP]); + candidate.green_writer_hg = row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP]) : -1; + candidate.green_reader_hg = row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP]) : -1; + candidate.max_lag_ms = atoi(row->fields[AWS_AURORA_MAX_LAG_MS]); + candidate.check_interval_ms = atoi(row->fields[AWS_AURORA_CHECK_INTERVAL_MS]); + candidate.check_timeout_ms = atoi(row->fields[AWS_AURORA_CHECK_TIMEOUT_MS]); + candidate.add_lag_ms = atoi(row->fields[AWS_AURORA_ADD_LAG_MS]); + candidate.min_lag_ms = atoi(row->fields[AWS_AURORA_MIN_LAG_MS]); + candidate.lag_num_checks = atoi(row->fields[AWS_AURORA_LAG_NUM_CHECKS]); + candidate.autopurge_missing_checks = + atoi(row->fields[AWS_AURORA_AUTOPURGE_MISSING_CHECKS]); + candidate.domain_name = row->fields[AWS_AURORA_DOMAIN_NAME] + ? row->fields[AWS_AURORA_DOMAIN_NAME] : ""; + candidate.writer_is_also_reader = + atoi(row->fields[AWS_AURORA_WRITER_IS_ALSO_READER]); + candidate.new_reader_weight = atoi(row->fields[AWS_AURORA_NEW_READER_WEIGHT]); + first_row = false; + } + + candidate.production_probe_hosts.push_back(AWS_RDS_BGD_Probe_Host { + row->fields[AWS_AURORA_HOSTNAME], + atoi(row->fields[AWS_AURORA_PORT]), + atoi(row->fields[AWS_AURORA_USE_SSL]), + }); + } + + if (candidate.production_probe_hosts.empty()) { + return false; + } + candidate.target_use_ssl = candidate.production_probe_hosts.front().use_ssl; + return true; +} + +bool MySQL_Monitor::aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at +) { + AWS_Aurora_BGD_State candidate; + if (!aws_aurora_bgd_load_worker_config(st.writer_hg, current_checksum, candidate)) { + return false; + } + + st.reader_hg = candidate.reader_hg; + st.green_writer_hg = candidate.green_writer_hg; + st.green_reader_hg = candidate.green_reader_hg; + st.max_lag_ms = candidate.max_lag_ms; + st.check_interval_ms = candidate.check_interval_ms; + st.check_timeout_ms = candidate.check_timeout_ms; + st.add_lag_ms = candidate.add_lag_ms; + st.min_lag_ms = candidate.min_lag_ms; + st.lag_num_checks = candidate.lag_num_checks; + st.autopurge_missing_checks = candidate.autopurge_missing_checks; + st.writer_is_also_reader = candidate.writer_is_also_reader; + st.new_reader_weight = candidate.new_reader_weight; + st.domain_name = candidate.domain_name; + st.production_probe_hosts = std::move(candidate.production_probe_hosts); + if (st.status == AWS_Aurora_BGD_Status::NONE && !st.target_snapshot_complete) { + st.target_use_ssl = candidate.target_use_ssl; + } + + next_loop_at = 0; + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: applied checksum %llu with in-place refresh at %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_aurora_bgd_status_str(st.status)); + return true; +} + void * monitor_AWS_Aurora_thread_HG(void *arg) { - unsigned int wHG = *(unsigned int *)arg; - unsigned int rHG = 0; - unsigned int num_hosts = 0; + AWS_Aurora_BGD_Worker* worker = static_cast(arg); + unsigned int wHG = worker->writer_hg; unsigned int cur_host_idx = 0; - unsigned int max_lag_ms = 0; - unsigned int check_interval_ms = 0; - unsigned int check_timeout_ms = 0; - unsigned int add_lag_ms = 0; - unsigned int min_lag_ms = 0; - unsigned int lag_num_checks = 1; - unsigned int autopurge_missing_checks = 0; - std::string domain_name; std::map autopurge_counter; AWS_Aurora_BGD_State bgd_state; bgd_state.writer_hg = wHG; @@ -6753,8 +6890,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t initial_raw_checksum = 0; - // this is a static array of the latest reads unsigned int ase_idx = 0; AWS_Aurora_status_entry *lasts_ase[N_L_ASE]; @@ -6762,78 +6897,13 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { lasts_ase[i] = NULL; } - // initial data load - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - initial_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - // count the number of hosts - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - num_hosts++; - if (max_lag_ms == 0) { - max_lag_ms = atoi(r->fields[5]); - } - if (check_interval_ms == 0) { - check_interval_ms = atoi(r->fields[6]); - } - if (check_timeout_ms == 0) { - check_timeout_ms = atoi(r->fields[7]); - } - if (rHG == 0) { - rHG = atoi(r->fields[1]); - } - add_lag_ms = atoi(r->fields[8]); - min_lag_ms = atoi(r->fields[9]); - lag_num_checks = atoi(r->fields[10]); - autopurge_missing_checks = atoi(r->fields[11]); - if (domain_name.empty() && r->fields[12]) { - domain_name = r->fields[12]; - } - if (r->fields[13] && r->fields[13][0]) { - bgd_state.green_writer_hg = atoi(r->fields[13]); - } - if (r->fields[14] && r->fields[14][0]) { - bgd_state.green_reader_hg = atoi(r->fields[14]); - } - } - } - host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*num_hosts); - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - hpa[cur_host_idx].host = strdup(r->fields[2]); - hpa[cur_host_idx].port = atoi(r->fields[3]); - hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); - bgd_state.production_probe_hosts.push_back({ - r->fields[2], - atoi(r->fields[3]), - atoi(r->fields[4]), - }); - cur_host_idx++; - } - } - bgd_state.reader_hg = rHG; - bgd_state.check_interval_ms = check_interval_ms; - bgd_state.check_timeout_ms = check_timeout_ms; - bgd_state.domain_name = domain_name; - if (!bgd_state.production_probe_hosts.empty()) { - bgd_state.target_use_ssl = bgd_state.production_probe_hosts.front().use_ssl; - } - // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access - // can table place later when accessing 'hpa[cur_host_idx]'. - if (cur_host_idx >= num_hosts) { - cur_host_idx = num_hosts - 1; - } - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - bool exit_now = false; unsigned long long t1 = 0; //unsigned long long t2 = 0; unsigned long long next_loop_at = 0; bool crc = false; - uint64_t current_raw_checksum = 0; + uint64_t last_checksum = 0; size_t rnd; bool found_pingable_host = false; bool rc_ping = false; @@ -6842,7 +6912,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { t1 = monotonic_time(); unsigned long long start_time=t1; - while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true + && worker->worker_stop.load()==false) { unsigned int glover; @@ -6864,14 +6935,17 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { next_loop_at=0; } - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - current_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - if (current_raw_checksum != initial_raw_checksum) { - // the content of AWS_Aurora_Hosts_resultset has changed. Exit - exit_now=true; - break; + uint64_t current_checksum = worker->current_checksum.load(); + if (current_checksum != last_checksum) { + if (!GloMyMon->aws_aurora_bgd_refresh_worker_config( + bgd_state, current_checksum, next_loop_at)) { + usleep(50000); + continue; + } + last_checksum = current_checksum; + if (cur_host_idx >= bgd_state.production_probe_hosts.size()) { + cur_host_idx = 0; + } } //fprintf(stderr,"%u : %llu %llu\n", wHG, t1, next_loop_at); if (t1 < next_loop_at) { @@ -6889,45 +6963,38 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { delete mmsd; mmsd = NULL; } - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } //proxy_info("Running check AWS Aurora writer HG %u\n", wHG); found_pingable_host = false; - rc_ping = false; - // pick a random host - rnd = (size_t) rand(); - rnd %= num_hosts; - rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); - //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); + const size_t num_hosts = bgd_state.production_probe_hosts.size(); + if (num_hosts != 0) { + rnd = static_cast(rand()) % num_hosts; + for (size_t offset = 0; !found_pingable_host && offset < num_hosts; ++offset) { + const size_t host_idx = (rnd + offset) % num_hosts; + const AWS_RDS_BGD_Probe_Host& host = + bgd_state.production_probe_hosts[host_idx]; + rc_ping = GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port); #ifdef TEST_AURORA_RANDOM - if (rand() % 100 < 30) { - // we randomly fail 30% of the requests - rc_ping = false; - } + if (offset == 0 && rand() % 100 < 30) { + rc_ping = false; + } #endif // TEST_AURORA_RANDOM - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = rnd; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - // the randomly picked host didn't work work - shuffle_hosts(hpa,num_hosts); - for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); if (rc_ping) { found_pingable_host = true; - cur_host_idx = i; + cur_host_idx = host_idx; } else { MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); + p_mysql_error_type::proxysql, wHG, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV); } } } @@ -6941,25 +7008,31 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } #ifdef TEST_AURORA if (rand() % 1000 == 0) { // suppress 99.9% of the output, too verbose - proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG , hpa[cur_host_idx].host, hpa[cur_host_idx].port); + const AWS_RDS_BGD_Probe_Host& host = bgd_state.production_probe_hosts[cur_host_idx]; + proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG, + host.hostname.c_str(), host.port); } #endif // TEST_AURORA if (mmsd) { delete mmsd; mmsd = NULL; } - //mmsd = NULL; - mmsd = new MySQL_Monitor_State_Data(MON_AWS_AURORA, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl); + const AWS_RDS_BGD_Probe_Host& probe_host = + bgd_state.production_probe_hosts[cur_host_idx]; + mmsd = new MySQL_Monitor_State_Data( + MON_AWS_AURORA, const_cast(probe_host.hostname.c_str()), + probe_host.port, probe_host.use_ssl); mmsd->writer_hostgroup = wHG; - mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; + mmsd->aws_aurora_check_timeout_ms = bgd_state.check_timeout_ms; mmsd->mysql=GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); //unsigned long long start_time=mysql_thr->curtime; start_time=t1; @@ -7018,7 +7091,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -7039,7 +7112,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -7053,7 +7126,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { __exit_monitor_aws_aurora_HG_thread: mmsd->t2=monotonic_time(); - next_loop_at = t1 + (check_interval_ms * 1000); + next_loop_at = t1 + (bgd_state.check_interval_ms * 1000); if (mmsd->t2 > t1) { next_loop_at -= (mmsd->t2 - t1); } @@ -7098,12 +7171,19 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { delete l_ase; } lasts_ase[ase_idx] = ase_l; - GloMyMon->evaluate_aws_aurora_results(wHG, rHG, &lasts_ase[0], ase_idx, max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks); + GloMyMon->evaluate_aws_aurora_results( + wHG, bgd_state.reader_hg, &lasts_ase[0], ase_idx, + bgd_state.max_lag_ms, bgd_state.add_lag_ms, + bgd_state.min_lag_ms, bgd_state.lag_num_checks); // Auto-purge servers that disappear from REPLICA_HOST_STATUS // Only process if autopurge is enabled and query was successful with results - if (autopurge_missing_checks > 0 && mmsd->interr == 0 && ase->host_statuses->size() > 0) { - GloMyMon->aws_aurora_autopurge_servers(wHG, rHG, ase, autopurge_missing_checks, autopurge_counter, domain_name); + if (bgd_state.autopurge_missing_checks > 0 && mmsd->interr == 0 + && ase->host_statuses->size() > 0) { + GloMyMon->aws_aurora_autopurge_servers( + wHG, bgd_state.reader_hg, ase, + bgd_state.autopurge_missing_checks, autopurge_counter, + bgd_state.domain_name); } for (auto h : *(ase_l->host_statuses)) { @@ -7181,14 +7261,19 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } - if (GloMyMon->shutdown == false && exit_now == false) { - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + if (GloMyMon->shutdown == false && worker->worker_stop.load() == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; } } __exit_monitor_AWS_Aurora_thread_HG_now: + if (bgd_state.status != AWS_Aurora_BGD_Status::NONE) { + GloMyMon->aws_aurora_bgd_apply_rollback( + bgd_state, AWS_Aurora_BGD_Status::NONE, true); + } if (mmsd) { delete (mmsd); mmsd = NULL; @@ -7200,7 +7285,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } - free(hpa); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; @@ -7217,9 +7301,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { void * MySQL_Monitor::monitor_aws_aurora() { - // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) - // Wait for GloMTH to be initialized - if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + if (!wait_for_glo_mth()) return NULL; unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime=monotonic_time(); @@ -7227,20 +7309,11 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); uint64_t last_raw_checksum = 0; - - // ADD here an unordered map , Writer HG => next time at - // when empty, a new map is populated - // when next_loop_at = 0 , the tables is emptied so to be populated again - - unsigned int *hgs_array = NULL; - pthread_t *pthreads_array = NULL; - unsigned int hgs_num = 0; + std::unordered_map> workers; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { - unsigned int glover; - - if (!GloMTH) return NULL; // quick exit during shutdown/restart + if (!GloMTH) break; // if variables has changed, triggers new checks glover=GloMTH->get_global_version(); @@ -7249,69 +7322,96 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); } - // if list of servers or HG or options has changed, triggers new checks + uint64_t new_raw_checksum = 0; + std::unordered_map cluster_checksums; pthread_mutex_lock(&aws_aurora_mutex); - uint64_t new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (AWS_Aurora_Hosts_resultset) { + new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (new_raw_checksum != last_raw_checksum) { + std::unordered_map> cluster_results; + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + const int writer_hg = atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]); + auto cluster_it = cluster_results.find(writer_hg); + if (cluster_it == cluster_results.end()) { + cluster_it = cluster_results.emplace( + writer_hg, std::unique_ptr( + new SQLite3_result(AWS_AURORA_HOSTS_COLUMNS))).first; + } + cluster_it->second->add_row(row); + } + for (const auto& [writer_hg, result] : cluster_results) { + cluster_checksums[writer_hg] = result->raw_checksum(); + } + } + } pthread_mutex_unlock(&aws_aurora_mutex); + if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS Aurora monitoring\n"); - if (pthreads_array) { - // wait all threads to terminate - for (unsigned int i=0; i < hgs_num; i++) { - pthread_join(pthreads_array[i], NULL); - proxy_info("Stopped Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - } - free(pthreads_array); - free(hgs_array); - pthreads_array = NULL; - hgs_array = NULL; - } - hgs_num = 0; - pthread_mutex_lock(&aws_aurora_mutex); - // Configuration can change while the previous workers are being joined. - // Launch replacements from the current snapshot and retain that checksum, - // so the coordinator never waits on a worker created from newer state. - last_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); - // scan all the writer HGs - unsigned int num_rows = AWS_Aurora_Hosts_resultset->rows_count; - if (num_rows) { - unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (std::vector::iterator it = AWS_Aurora_Hosts_resultset->rows.begin() ; it != AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int wHG = atoi(r->fields[0]); - bool found = false; - // very simple search. Far from optimal, but assuming very few HGs it is fast enough - for (unsigned int i=0; i < hgs_num; i++) { - if (tmp_hgs_array[i] == (unsigned int)wHG) { - found = true; - } - } - if (found == false) { - // new wHG found - tmp_hgs_array[hgs_num]=wHG; - hgs_num++; - } + last_raw_checksum = new_raw_checksum; + std::vector stopped_workers; + + for (auto& [writer_hg, worker] : workers) { + auto cluster_it = cluster_checksums.find(writer_hg); + if (cluster_it == cluster_checksums.end()) { + worker->worker_stop.store(true); + stopped_workers.push_back(writer_hg); + proxy_info( + "AWS Aurora BGD [wHG=%d]: stopping worker; owning row is inactive, removed, or has no monitored server\n", + writer_hg); + continue; } - proxy_info("Activating Monitoring of %u AWS Aurora clusters\n", hgs_num); - hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*hgs_num); - pthreads_array = (pthread_t *)malloc(sizeof(pthread_t)*hgs_num); - for (unsigned int i=0; i < hgs_num; i++) { - hgs_array[i] = tmp_hgs_array[i]; - proxy_info("Starting Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - if (pthread_create(&pthreads_array[i], NULL, monitor_AWS_Aurora_thread_HG, &hgs_array[i]) != 0) { - // LCOV_EXCL_START - proxy_error("Thread creation\n"); - assert(0); - // LCOV_EXCL_STOP - } + + uint64_t old_checksum = worker->current_checksum.load(); + if (old_checksum != cluster_it->second) { + worker->current_checksum.store(cluster_it->second); + proxy_info( + "AWS Aurora BGD [wHG=%d]: signaling config refresh, checksum %llu -> %llu\n", + writer_hg, (unsigned long long)old_checksum, + (unsigned long long)cluster_it->second); } - free(tmp_hgs_array); } - pthread_mutex_unlock(&aws_aurora_mutex); + + for (const auto& [writer_hg, checksum] : cluster_checksums) { + if (workers.find(writer_hg) != workers.end()) { + continue; + } + std::unique_ptr worker(new AWS_Aurora_BGD_Worker); + worker->writer_hg = writer_hg; + worker->current_checksum.store(checksum); + AWS_Aurora_BGD_Worker* worker_arg = worker.get(); + workers.emplace(writer_hg, std::move(worker)); + proxy_info("Starting Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + if (pthread_create( + &worker_arg->thread, NULL, monitor_AWS_Aurora_thread_HG, worker_arg) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + + for (int writer_hg : stopped_workers) { + auto worker_it = workers.find(writer_hg); + if (worker_it == workers.end()) { + continue; + } + pthread_join(worker_it->second->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + workers.erase(worker_it); + } } usleep(10000); } + for (auto& [writer_hg, worker] : workers) { + worker->worker_stop.store(true); + } + for (auto& [writer_hg, worker] : workers) { + pthread_join(worker->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + } + workers.clear(); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index cb6b8a8746..b8853c07d1 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -72,3 +72,8 @@ aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-reader-1-green-r8t4.b1.us-east-1.rds.amazonaws.com 127.0.12.22 aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 + +# Aurora blue/green deployment C supports three-cluster isolation scenarios. +aurora-c-writer.c1.us-east-1.rds.amazonaws.com 127.0.13.11 +aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com 127.0.13.20 +aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com 127.0.13.21 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 9afa842be7..c4de080fa4 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -366,6 +366,9 @@ "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_resilience-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_lifecycle-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index a38cb4bb5f..c71d8100c1 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -113,6 +113,38 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { return deployment; } +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment C writer-only"; + deployment.domain_name = ".c1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-c"; + deployment.target_replica_set = "aurora-bgd-target-c"; + deployment.source_topology_id = "aurora-bgd-source-c"; + deployment.target_topology_id = "aurora-bgd-target-c"; + deployment.target_cluster_endpoint = { + "aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com", "127.0.13.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-c-writer", "MASTER_SESSION_ID", + {"aurora-c-writer.c1.us-east-1.rds.amazonaws.com", "127.0.13.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-c-writer-green-m5n9", "MASTER_SESSION_ID", + {"aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com", "127.0.13.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + inline string aurora_bgd_sql_quote(const string& value) { string quoted {"'"}; for (char c : value) { diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp new file mode 100644 index 0000000000..743b5e4673 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp @@ -0,0 +1,306 @@ +/** + * @file test_aurora_bgd_cluster_sync-t.cpp + * @brief Aurora BGD configuration sync preserves worker-owned status per node. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; +namespace fs = std::filesystem; + +const uint32_t kWaitSeconds = 10; +const int kReplicaAdminPort = 16062; +const int kReplicaMySQLPort = 16063; +const char kReplicaHost[] = "127.0.0.1"; +const char kSQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; + +struct Replica_Process { + pid_t pid = -1; + string directory; + string config_path; + string stderr_path; +}; + +string config_quote(const string& value) { + string quoted; + for (char c : value) { + if (c == '\\' || c == '"') { + quoted += '\\'; + } + quoted += c; + } + return quoted; +} + +int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { + char directory_template[] = "/tmp/proxysql-aurora-bgd-sync-XXXXXX"; + char* directory = mkdtemp(directory_template); // NOSONAR: mkdtemp creates an owner-only directory. + if (directory == nullptr) { + diag("mkdtemp failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + process.directory = directory; + process.config_path = process.directory + "/proxysql.cnf"; + process.stderr_path = process.directory + "/proxysql.stderr"; + + ofstream config(process.config_path); + if (!config.is_open()) { + diag("failed to create replica config: %s", process.config_path.c_str()); + return EXIT_FAILURE; + } + config + << "datadir=\"" << config_quote(process.directory) << "\"\n" + << "admin_variables={\n" + << " admin_credentials=\"" << config_quote(cl.admin_username) << ":" + << config_quote(cl.admin_password) << ";radmin:radmin\"\n" + << " mysql_ifaces=\"0.0.0.0:" << kReplicaAdminPort << "\"\n" + << " cluster_username=\"radmin\"\n" + << " cluster_password=\"radmin\"\n" + << " cluster_check_interval_ms=200\n" + << " cluster_check_status_frequency=100\n" + << " cluster_admin_variables_diffs_before_sync=0\n" + << " cluster_mysql_servers_diffs_before_sync=1\n" + << " cluster_mysql_servers_save_to_disk=false\n" + << " cluster_mysql_servers_sync_algorithm=3\n" + << "}\n" + << "mysql_variables={\n" + << " interfaces=\"0.0.0.0:" << kReplicaMySQLPort << "\"\n" + << " monitor_username=\"aurora1\"\n" + << " monitor_password=\"pass1\"\n" + << " monitor_connect_timeout=500\n" + << " monitor_ping_interval=10000\n" + << "}\n" + << "proxysql_servers=()\n"; + config.close(); + return config.fail() ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int launch_replica(const CommandLine& cl, Replica_Process& process) { + if (prepare_replica_config(cl, process) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + const string binary = string(cl.workdir) + "../../../src/proxysql"; + process.pid = fork(); + if (process.pid == -1) { + diag("fork failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + if (process.pid == 0) { + int stderr_fd = open(process.stderr_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (stderr_fd >= 0) { + dup2(stderr_fd, STDOUT_FILENO); + dup2(stderr_fd, STDERR_FILENO); + close(stderr_fd); + } + execl( + binary.c_str(), "proxysql", "--sqlite3-server", "-f", "-c", + process.config_path.c_str(), static_cast(nullptr)); + _exit(127); + } + return EXIT_SUCCESS; +} + +void stop_replica(MYSQL*& admin, Replica_Process& process, bool preserve_log) { + if (admin != nullptr) { + mysql_query(admin, "PROXYSQL SHUTDOWN"); + mysql_close(admin); + admin = nullptr; + } + if (process.pid > 0) { + bool exited = false; + for (int i = 0; i < 50; ++i) { + pid_t rc = waitpid(process.pid, nullptr, WNOHANG); + if (rc == process.pid || rc == -1) { + exited = true; + break; + } + usleep(100000); + } + if (!exited) { + kill(process.pid, SIGKILL); + waitpid(process.pid, nullptr, 0); + } + } + if (!preserve_log && !process.directory.empty()) { + fs::remove_all(process.directory); + } else if (preserve_log) { + diag("replica ProxySQL log retained at %s", process.stderr_path.c_str()); + } +} + +Aurora_BGD_Test_Deployment peer_deployment() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD peer-local status"; + deployment.domain_name = ".localhost"; + deployment.blue_replica_set = "aurora-bgd-peer-blue"; + deployment.target_replica_set = "aurora-bgd-peer-target"; + deployment.source_topology_id = "aurora-bgd-peer-source"; + deployment.target_topology_id = "aurora-bgd-peer-target"; + deployment.target_cluster_endpoint = { + "aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307 + }; + deployment.production = { + deployment.blue_replica_set, + {aurora_bgd_member( + "aurora-peer-writer", "MASTER_SESSION_ID", + {"aurora-peer-writer.localhost", "127.0.0.1", 3306})}, + {} + }; + deployment.production.serving_endpoints.push_back( + deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + {aurora_bgd_member( + "aurora-peer-writer-green-sync", "MASTER_SESSION_ID", + {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307})}, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back( + deployment.target.members.front().endpoint); + return deployment; +} + +int configure_sqlite_interfaces(MYSQL* admin, const string& interfaces) { + return aurora_bgd_execute_all(admin, { + "SET sqliteserver-mysql_ifaces=" + aurora_bgd_sql_quote(interfaces), + "LOAD SQLITESERVER VARIABLES TO RUNTIME", + }); +} + +int configure_peer( + MYSQL* admin, BGD_Simulator& simulator, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return simulator.cleanup() == EXIT_SUCCESS + && aurora_bgd_publish(simulator, deployment) == EXIT_SUCCESS + && simulator.topology_update( + aurora_bgd_topology_backends(deployment), aurora_bgd_topology(deployment, status)) + == EXIT_SUCCESS + && aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && aurora_bgd_admin_setup(admin, deployment, 1800, 1801, 1802, 1803, false) + == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int main() { + plan(5); + CommandLine cl {}; + if (cl.getEnv()) { + diag("failed to load TAP environment"); + return exit_status(); + } + + MYSQL* primary_admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + MYSQL* replica_admin = nullptr; + Replica_Process replica_process; + string primary_sqlite_interfaces; + BGD_Simulator primary_simulator; + BGD_Simulator replica_simulator; + Aurora_BGD_Test_Deployment deployment = peer_deployment(); + bool primary_simulator_connected = false; + bool replica_simulator_connected = false; + + if (primary_admin == nullptr + || get_variable_value( + primary_admin, "sqliteserver-mysql_ifaces", primary_sqlite_interfaces) != EXIT_SUCCESS + || configure_sqlite_interfaces(primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS + || launch_replica(cl, replica_process) != EXIT_SUCCESS) { + diag("failed to prepare the two ProxySQL nodes"); + goto cleanup; + } + + replica_admin = wait_for_proxysql( + {kReplicaHost, cl.admin_username, cl.admin_password, kReplicaAdminPort}, + kWaitSeconds); + if (replica_admin == nullptr + || configure_sqlite_interfaces(replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { + diag("failed to start the replica ProxySQL node"); + goto cleanup; + } + + { + char username[] = "aurora1"; + char password[] = "pass1"; + primary_simulator_connected = primary_simulator.connect( + cl.host, 3306, username, password) == EXIT_SUCCESS; + replica_simulator_connected = replica_simulator.connect( + const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; + } + if (!primary_simulator_connected || !replica_simulator_connected + || configure_peer(primary_admin, primary_simulator, deployment, "AVAILABLE") + != EXIT_SUCCESS + || configure_peer( + replica_admin, replica_simulator, deployment, "SWITCHOVER_INITIATED") + != EXIT_SUCCESS) { + diag("failed to publish the two node-local Aurora observations"); + goto cleanup; + } + + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "primary worker publishes its local AVAILABLE observation"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "replica worker publishes its local SWITCHOVER_INITIATED observation"); + + if (aurora_bgd_execute_all(replica_admin, { + "DELETE FROM proxysql_servers", + "INSERT INTO proxysql_servers(hostname,port,weight,comment) VALUES (" + + aurora_bgd_sql_quote(cl.admin_host) + "," + to_string(cl.admin_port) + + ",0,'Aurora BGD sync primary')", + "LOAD PROXYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS + || aurora_bgd_execute_all(primary_admin, { + "UPDATE mysql_aws_aurora_hostgroups SET comment='peer-config-synced' " + "WHERE writer_hostgroup=1800", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("failed to initiate Aurora configuration synchronization"); + goto cleanup; + } + + ok(wait_for_cond( + replica_admin, + "SELECT COUNT(*)=1 FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1800 AND comment='peer-config-synced'", + kWaitSeconds) == EXIT_SUCCESS, + "Aurora BGD configured fields synchronize to the peer"); + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "configuration sync preserves the primary's local status"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "configuration sync preserves the replica's local status"); + +cleanup: + if (primary_simulator_connected) { + primary_simulator.cleanup(); + } + if (replica_simulator_connected) { + replica_simulator.cleanup(); + } + if (replica_admin != nullptr) { + aurora_bgd_admin_cleanup(replica_admin); + } + if (primary_admin != nullptr) { + aurora_bgd_admin_cleanup(primary_admin); + if (!primary_sqlite_interfaces.empty()) { + configure_sqlite_interfaces(primary_admin, primary_sqlite_interfaces); + } + mysql_close(primary_admin); + } + stop_replica(replica_admin, replica_process, tests_failed() != 0); + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp new file mode 100644 index 0000000000..77a2e210d0 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp @@ -0,0 +1,397 @@ +/** + * @file test_aurora_bgd_lifecycle-t.cpp + * @brief Aurora BGD reload, worker removal, and concurrent-cluster behavior. + */ + +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_writer_route( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD lifecycle route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_writer( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + return route_to_backend(cl, sim, expected); +} + +bool wait_for_writer_route( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { + if (route_to_backend(cl, sim, expected)) { + return true; + } + usleep(100000); + } + return false; +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", + kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg), + kWaitSeconds) == EXIT_SUCCESS; +} + +int main() { + plan(28); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment reload = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, reload) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, reload, 1620, 1621, 1622, 1623, false, 300) + != EXIT_SUCCESS + || add_writer_route(admin, reload, 1624) != EXIT_SUCCESS) { + diag("Error: failed to configure the active reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1620, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "reload scenario starts from AVAILABLE"); + ok(publish_status(sim, reload, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, reload, 1624, true), + "active reload scenario pins writer traffic to the target"); + ok(aurora_bgd_execute_all(admin, {"LOAD MYSQL SERVERS TO RUNTIME"}) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "an unrelated server reload preserves active FSM status"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "an unrelated server reload preserves the applied traffic pin"); + ok(aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "disabling auto-discovery does not abort an admitted deployment"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "a variable refresh preserves cached membership and target routing"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1626," + "green_reader_hostgroup=1627 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND green_writer_hostgroup=1626 " + "AND green_reader_hostgroup=1627 " + "AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", + kWaitSeconds) == EXIT_SUCCESS, + "green hostgroup changes refresh staging references without resetting the FSM"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "green hostgroup refresh preserves applied pins and cached target IPs"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS && wait_for_inactive_none(admin, 1620), + "deactivating the owning row stops the worker after publishing NONE"); + ok(writer_placement( + admin, 1620, 1621, reload.production.members.front().endpoint.hostname, false), + "worker teardown restores canonical writer placement"); + ok(route_writer(cl, admin, sim, reload, 1624, false), + "worker teardown removes the applied traffic pin"); + ok(wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND active=0)=1) AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostname=" + + aurora_bgd_sql_quote(reload.production.members.front().endpoint.hostname) + ")>=1)", + kWaitSeconds) == EXIT_SUCCESS, + "teardown preserves user configuration and server rows"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment terminal = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, terminal) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, terminal, 1630, 1631, -1, -1, true, 300) + != EXIT_SUCCESS) { + diag("Error: failed to configure the terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1630, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "terminal reload scenario starts from AVAILABLE"); + ok(sim.topology_update( + aurora_bgd_topology_backends(terminal), aurora_bgd_completed_topology(terminal)) + == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1630, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "direct completion enters the terminal latch"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1632," + "green_reader_hostgroup=1633 WHERE writer_hostgroup=1630", + "LOAD MYSQL SERVERS TO RUNTIME", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1630 AND green_writer_hostgroup=1632 " + "AND green_reader_hostgroup=1633 AND bgd_status='SWITCHOVER_COMPLETED'", + kWaitSeconds) == EXIT_SUCCESS, + "configuration and variable refresh preserve the terminal latch"); + ok(sim.topology_delete(aurora_bgd_topology_backends(terminal)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1630, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful topology drain rearms a refreshed terminal worker"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before concurrent scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment concurrent_a = aurora_bgd_deployment_a(); + Aurora_BGD_Test_Deployment concurrent_b = aurora_bgd_deployment_b_writer_only(); + Aurora_BGD_Test_Deployment concurrent_c = aurora_bgd_deployment_c_writer_only(); + if (aurora_bgd_publish(sim, concurrent_a) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_b) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_c) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_a, 1640, 1641, 1642, 1643, false, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_b, 1650, 1651, -1, -1, true, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_c, 1660, 1661, -1, -1, true, 300) + != EXIT_SUCCESS + || add_writer_route(admin, concurrent_a, 1644) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_b, 1654) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_c, 1664) != EXIT_SUCCESS) { + diag("Error: failed to configure three concurrent deployments"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1640, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1650, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1660, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "three writer hostgroups discover deployments independently"); + ok(publish_status(sim, concurrent_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1650, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "three workers enter POST_PROCESSING without sharing FSM state"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true), + "deployment A owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_b, 1654, true), + "deployment B owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment C owns its target pin"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1650", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_inactive_none(admin, 1650) + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "deactivating deployment B cleans only its worker state"); + ok(writer_placement( + admin, 1650, 1651, concurrent_b.production.members.front().endpoint.hostname, false) + && route_writer(cl, admin, sim, concurrent_b, 1654, false), + "deployment B teardown restores only its production routing"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment B teardown leaves A and C pins intact"); + ok(sim.topology_update( + aurora_bgd_topology_backends(concurrent_a), + aurora_bgd_completed_topology(concurrent_a)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "completing deployment A leaves deployment C active"); + ok(add_writer_route(admin, concurrent_a, 1645) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment A cleanup removes only its pin"); + ok(aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=1660", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_runtime_row_absent(admin, 1660) + && writer_placement( + admin, 1660, 1661, + concurrent_c.production.members.front().endpoint.hostname, false) + && wait_for_writer_route(cl, admin, sim, concurrent_c, 1664, false), + "removing deployment C safely restores its production routing"); + ok(aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false), + "deployment C removal leaves deployment A terminal state unchanged"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD lifecycle test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_resilience-t.cpp b/test/tap/tests/test_aurora_bgd_resilience-t.cpp new file mode 100644 index 0000000000..9d11f19728 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_resilience-t.cpp @@ -0,0 +1,412 @@ +/** + * @file test_aurora_bgd_resilience-t.cpp + * @brief Aurora BGD rollback, error retention, and active-state late entry. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_initial( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool active_probe_policy( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool membership = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set; + } + return membership; +} + +bool wait_for_ordinary_probe( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, log] = aurora_bgd_wait_for_replica_probe( + sim, sequence, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int add_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, int green_writer_hg +) { + vector queries; + for (size_t i = 0; i < route_hgs.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback route')"); + } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback green pool')"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " + "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; +} + +int main() { + plan(20); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment initiated = aurora_bgd_deployment_b_writer_only(); + const int initiated_writer_hg = 1590; + const int initiated_reader_hg = 1591; + if (publish_initial(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, initiated, initiated_writer_hg, initiated_reader_hg, + 1592, 1593, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "late entry at INITIATED publishes the observed state"); + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "late INITIATED entry leaves writer placement unchanged"); + auto [initiated_seq_rc, initiated_sequence] = sim.replica_probe_log_last_sequence(); + ok(initiated_seq_rc == EXIT_SUCCESS && active_probe_policy( + sim, initiated_sequence, initiated.target_replica_set, 450), + "late INITIATED entry reconstructs fast membership probing"); + auto [topology_error_seq_rc, topology_error_sequence] = sim.probe_log_last_sequence(); + auto topology_error_seen = topology_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(initiated), 1205, "simulated topology timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, topology_error_sequence, initiated.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(topology_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "topology errors retain the active state"); + if (publish_status(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + diag("Error: failed to restore topology after error"); + cleanup(admin, sim); + return exit_status(); + } + auto [membership_error_seq_rc, membership_error_sequence] = + sim.replica_probe_log_last_sequence(); + auto membership_error_seen = membership_error_seq_rc == EXIT_SUCCESS + && sim.replica_error( + initiated.target.backends(), 1205, "simulated membership timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, membership_error_sequence, initiated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(membership_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "membership errors retain the last complete state"); + if (sim.replica_update( + initiated.target.replica_set_id, initiated.target.replica_rows(), + initiated.target.backends()) != EXIT_SUCCESS + || publish_status(sim, initiated, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance to IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, true), + "IN_PROGRESS demotes the reconstructed production writer"); + ok(publish_status(sim, initiated, "SWITCHOVER_INITIATED") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "a backward status rolls back writer placement before entering the earlier state"); + auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); + ok(sim.topology_delete(aurora_bgd_topology_backends(initiated)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful empty topology cancels the active deployment"); + ok(ordinary_seq_rc == EXIT_SUCCESS + && wait_for_ordinary_probe(sim, ordinary_sequence, initiated), + "cancellation resumes ordinary production probing"); + ok(publish_status(sim, initiated, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "the worker admits a repeated deployment after cancellation"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + if (publish_initial(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, progress, 1600, 1601, 1602, 1603, false, 300, false) + != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1600, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, true), + "late entry at IN_PROGRESS reconstructs and demotes the writer"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1600, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, false), + "confirmed topology absence rolls back IN_PROGRESS to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const vector route_hgs {1614, 1615, 1616}; + if (publish_initial(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, post, 1610, 1611, 1612, 1613, false, 300, false) + != EXIT_SUCCESS + || add_routes(admin, post, route_hgs, 1612) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "late entry at POST_PROCESSING reconstructs the active phase"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "late POST_PROCESSING entry reconstructs and applies every target pin"); + ok(set_default_hostgroup(admin, 1612) == EXIT_SUCCESS + && route_to_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, 1612) >= 1, + "a configured green pool is established before rollback"); + auto [post_error_seq_rc, post_error_sequence] = sim.probe_log_last_sequence(); + bool post_error_seen = post_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated post timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, post_error_sequence, post.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_error_seen && aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "topology errors do not roll back applied POST_PROCESSING pins"); + if (publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to restore POST_PROCESSING topology"); + cleanup(admin, sim); + return exit_status(); + } + auto [post_membership_seq_rc, post_membership_sequence] = + sim.replica_probe_log_last_sequence(); + bool post_membership_error = post_membership_seq_rc == EXIT_SUCCESS + && sim.replica_error(post.target.backends(), 1205, "simulated post membership timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, post_membership_sequence, post.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_membership_error + && sim.replica_update( + post.target.replica_set_id, post.target.replica_rows(), post.target.backends()) + == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "membership errors retain the last complete mapped routing"); + ok(publish_status(sim, post, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1610, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "a backward POST_PROCESSING status completes rollback before AVAILABLE"); + ok(writer_placement( + admin, 1610, 1611, post.production.members.front().endpoint.hostname, false) + && route_members(cl, admin, sim, post, route_hgs, false), + "rollback removes pins and restores canonical writer and member routing"); + ok(pool_count(admin, 1612) >= 1, + "rollback preserves configured green pools instead of draining them"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD resilience test data"); + return EXIT_FAILURE; + } + return exit_status(); +}