test(pg-compat): SP-2 — polyglot PG test foundation (dbdeployer-PG infra, Toxiproxy, differential engine, routing oracle, CI)

.github/workflows/CI-pg-compat.yml

@@ -0,0 +1,47 @@
+# PAIRED FILE -- caller half of the CI-pg-compat pair (lives on v3.0). The
+# reusable half, .github/workflows/gh-actions-reusable/ci-pg-compat.yml, is
+# staged in THIS repo for review but must be merged to the `GH-Actions`
+# branch FIRST, at path `.github/workflows/ci-pg-compat.yml`, before (never
+# after) this caller file merges to v3.0 -- see doc/GH-Actions/README.md
+# "Merge order" (~line 816): `workflow_run`/`workflow_call` references are
+# only resolved against files that already exist on the target branch, so a
+# caller landing before its reusable exists on GH-Actions fails immediately
+# with "Unable to resolve action". See doc/GH-Actions/README.md (~lines
+# 42-170) for the full two-branch caller/reusable split rationale.
+name: CI-pg-compat
+
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  pg-compat:
+    # Runs on: the nightly schedule, a manual dispatch, or a pull_request
+    # that carries the 'pg-compat' label (checked on every listed pull_request
+    # type, including 'labeled', so adding the label to an already-open PR
+    # triggers a run without needing a new commit). Unlike the TAP families,
+    # this does NOT chain off CI-trigger/CI-builds -- it builds ProxySQL
+    # inline in the reusable job, so it doesn't need CI-builds' cache to
+    # exist first (nightly/label runs have no guaranteed prior CI-builds run
+    # to restore from).
+    if: >-
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'pg-compat')
+    # write-all: reusable-workflow permissions are the intersection of
+    # caller + callee: the callee at ci-pg-compat.yml@GH-Actions also
+    # declares write-all (needed for actions/upload-artifact's write scope
+    # under the pull_request event, matching CI-3p-postgresql.yml's
+    # documented rationale).
+    permissions: write-all
+    uses: sysown/proxysql/.github/workflows/ci-pg-compat.yml@GH-Actions
+    secrets: inherit
+    with:
+      trigger: ${{ toJson(github) }}

.github/workflows/gh-actions-reusable/ci-pg-compat.yml

@@ -0,0 +1,124 @@
+# STAGED FILE -- this is the reusable half of the CI-pg-compat pair.
+# It is authored here (on v3.0, under gh-actions-reusable/) for review, but
+# it does NOT run from here. It must be merged to the `GH-Actions` branch
+# at path `.github/workflows/ci-pg-compat.yml` FIRST, before (or in the same
+# merge window as, but not after) the caller `.github/workflows/CI-pg-compat.yml`
+# lands on `v3.0` -- see doc/GH-Actions/README.md "Merge order" (~line 816):
+# a caller referencing `ci-pg-compat.yml@GH-Actions` before that file exists
+# on GH-Actions fails immediately with "Unable to resolve action". See the
+# two-branch caller/reusable split explained in doc/GH-Actions/README.md
+# (~lines 42-170): callers (`CI-*.yml`, uppercase) live on `v3.0`; reusables
+# (`ci-*.yml`, lowercase) live on `GH-Actions`.
+name: CI-pg-compat
+
+on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      trigger:
+        type: string
+
+# No env.SHA/trigger-JSON parsing here (unlike the workflow_run-triggered
+# reusables, e.g. ci-legacy-g4.yml): those need it because their caller is
+# invoked BY workflow_run, whose own github.sha is the default branch tip,
+# not the real source commit -- the real sha only exists inside the passed
+# `trigger` JSON. This caller triggers directly via pull_request/schedule/
+# workflow_dispatch, so github.sha here (a workflow_call callee inherits the
+# caller's context) already IS the right commit; `inputs.trigger` is kept
+# only for parity with the sibling callers' `with: trigger: ...` shape and
+# isn't parsed for a sha. checkout below uses actions/checkout@v4's default
+# ref (the triggering ref), so no untrusted github.event.* field is ever
+# substituted into a `ref:`.
+
+jobs:
+  pg-compat:
+    runs-on: ubuntu-22.04
+    # Generous budget: a from-scratch `PROXYSQL31=1 make debug` (deps -> lib
+    # -> src) on a 2-core GH-hosted runner is the dominant cost here (there
+    # is no build-cache restore in this job, unlike the CI-builds-fed TAP
+    # families -- this suite runs inline, like the CI-3p-* family, since its
+    # schedule/label triggers have no guaranteed prior CI-builds run to
+    # restore a cache from).
+    timeout-minutes: 120
+    permissions: write-all
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Inline build (CI-3p-* model, not the CI-trigger/CI-builds cache-chain
+      # model used by the TAP families): no ccache pattern exists elsewhere
+      # in this repo's workflows (checked both branches) to reuse, so this
+      # is a plain build for v1. PROXYSQL31=1 is required -- bare `make`
+      # would leave FFTO/TSDB symbols out and is not what any tier actually
+      # ships; debug is required because the isolated harness
+      # (start-proxysql-isolated.bash / ensure-infras.bash) issues
+      # debug-only admin commands.
+      - name: Build ProxySQL (debug, PROXYSQL31)
+        run: PROXYSQL31=1 make -j$(nproc) debug
+
+      # Stand up the pg-compat infra: dbdeployer PG17 primary+2-replica
+      # backend, Toxiproxy sidecar, and the ProxySQL container built above
+      # (ensure-infras.bash starts ProxySQL itself via
+      # start-proxysql-isolated.bash if it isn't already running -- see
+      # test/infra/control/ensure-infras.bash step 2). Never manage Docker
+      # by hand here; this script is the only supported entry point.
+      - name: Stand up infra (backends + Toxiproxy + ProxySQL)
+        env:
+          INFRA_ID: ci-${{ github.run_id }}
+          WORKSPACE: ${{ github.workspace }}
+          TAP_GROUP: pg-compat
+        run: test/infra/control/ensure-infras.bash
+
+      # Non-gating (discovery phase, spec sec 2.1): the suite's job right now
+      # is to build a failure inventory in xfail.toml, not to be all-green.
+      # `|| true` keeps this step (and therefore the job) from failing the
+      # workflow on real/uncatalogued divergences during discovery. Promote
+      # to gating by dropping `|| true` (and tightening xfail.toml) once the
+      # suite is green and stable -- see test/pg-compat/README.md.
+      #
+      # --junitxml path: run-pg-compat.bash's container runs with --rm, so a
+      # report written to the container's own filesystem (e.g. /tmp) would
+      # be destroyed on exit and never reach this runner -- traced and fixed
+      # in run-pg-compat.bash, which now bind-mounts a host directory
+      # (default "${WORKSPACE}/pg-compat-reports", override via
+      # PGCOMPAT_REPORT_DIR) to /pg-compat-reports inside the container.
+      # Writing the report there is what makes it visible to the upload
+      # step below.
+      - name: Run pg-compat suite (non-gating, discovery phase)
+        env:
+          INFRA_ID: ci-${{ github.run_id }}
+          WORKSPACE: ${{ github.workspace }}
+        run: test/pg-compat/run-pg-compat.bash --junitxml=/pg-compat-reports/pg-compat.xml -rxX || true
+
+      # The pg-compat container's default user is root, so the bind-mounted
+      # report directory is root-owned on the host afterwards; chown it back
+      # to the runner user before upload-artifact (which runs as the
+      # non-root runner account) tries to read it. Same pattern already
+      # used for docker-written logs in ci-3p-postgresql.yml.
+      - name: Fix report ownership
+        if: always()
+        run: sudo chown -R "$(id -u):$(id -g)" "${{ github.workspace }}/pg-compat-reports" || true
+
+      - name: Publish report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pg-compat-report
+          path: ${{ github.workspace }}/pg-compat-reports/pg-compat.xml
+          if-no-files-found: warn
+
+      # Teardown always runs, mirroring ci-legacy-g4.yml's cleanup step:
+      # stop the ProxySQL container first, then tear down the backend +
+      # Toxiproxy infra. destroy-infras.bash is test/infra/control's
+      # documented teardown entry point (paired with ensure-infras.bash).
+      - name: Cleanup
+        if: always()
+        env:
+          INFRA_ID: ci-${{ github.run_id }}
+          WORKSPACE: ${{ github.workspace }}
+          TAP_GROUP: pg-compat
+        run: |
+          set +e
+          docker logs "proxysql.${INFRA_ID}" 2>&1 | tail -50 || true
+          test/infra/control/stop-proxysql-isolated.bash
+          test/infra/control/destroy-infras.bash

.gitignore

@@ -229,6 +229,10 @@ test/tap/tests/parsersql_digest_test
 test/tap/tests/setparser_parsersql_test
 deps/protobuf/protobuf-*/
 
+# pg-compat report output (run-pg-compat.bash's default host bind-mount
+# target; see test/pg-compat/README.md "Report output")
+/pg-compat-reports/
+
 # Stray build artifact: run-tests-isolated.bash creates the real mysqlbinlog
 # symlink at runtime under test-scripts/deps/; never commit this one (it was
 # accidentally committed once with an absolute /home path).

test/infra/infra-dbdeployer-pgsql17-repl/.env

@@ -0,0 +1,13 @@
+PGSQL_VERSION=17.10
+
+PREFIX=00
+
+WHG=00
+RHG=01
+
+# dbdeployer deploys all 3 PG nodes in ONE container; ports are auto-derived
+# from the version (15000 + major*100 + minor) and fixed for 17.10.
+PG_PRIMARY_HOST=dbdeployer1
+PG_PRIMARY_PORT=16710
+PG_REPLICA1_PORT=16711
+PG_REPLICA2_PORT=16712

test/infra/infra-dbdeployer-pgsql17-repl/bin/docker-pgsql-post.bash

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Host-side verification of the dbdeployer PostgreSQL replication backend.
+# Provisioning (roles/databases/extension) happens inside the container's
+# entrypoint; this script is VERIFICATION-ONLY. It asserts:
+#   - all 3 nodes are reachable and report the expected recovery state (f/t/t)
+#   - pg_stat_statements is queryable on every node
+#   - testuser can authenticate over TCP using its password
+set -e
+set -o pipefail
+[ -f .env ] && . .env
+
+CONTAINER="${COMPOSE_PROJECT}-dbdeployer1-1"
+
+PRIMARY_PORT="${PG_PRIMARY_PORT:-16710}"
+REPLICA1_PORT="${PG_REPLICA1_PORT:-16711}"
+REPLICA2_PORT="${PG_REPLICA2_PORT:-16712}"
+
+# psql runs INSIDE the container (image bakes psql onto PATH); connect over
+# loopback (trust) as postgres for the recovery/extension assertions.
+pg() {
+    local port="$1"; shift
+    docker exec "${CONTAINER}" psql -h 127.0.0.1 -p "${port}" -U postgres -d postgres -tAc "$1"
+}
+
+printf "[%s] PgSQL replication verification (Container: %s)\n" "$(date)" "${CONTAINER}"
+
+# 1. Reachability + recovery state (primary=f, replicas=t).
+declare -A EXPECT=( ["${PRIMARY_PORT}"]="f" ["${REPLICA1_PORT}"]="t" ["${REPLICA2_PORT}"]="t" )
+for PORT in "${PRIMARY_PORT}" "${REPLICA1_PORT}" "${REPLICA2_PORT}"; do
+    echo -n "  - node ${PORT}: waiting for connectivity..."
+    MAX_WAIT=60; COUNT=0
+    while ! pg "${PORT}" "SELECT 1" >/dev/null 2>&1; do
+        if [ $COUNT -ge $MAX_WAIT ]; then echo " TIMEOUT"; exit 1; fi
+        echo -n "."; sleep 2; COUNT=$((COUNT + 2))
+    done
+    REC=$(pg "${PORT}" "SELECT pg_is_in_recovery();")
+    if [ "${REC}" != "${EXPECT[$PORT]}" ]; then
+        echo " FAIL (pg_is_in_recovery=${REC}, expected ${EXPECT[$PORT]})"
+        exit 1
+    fi
+    echo " OK (pg_is_in_recovery=${REC})"
+done
+
+# 2. pg_stat_statements queryable on every node.
+for PORT in "${PRIMARY_PORT}" "${REPLICA1_PORT}" "${REPLICA2_PORT}"; do
+    echo -n "  - node ${PORT}: pg_stat_statements..."
+    if ! pg "${PORT}" "SELECT count(*) FROM pg_stat_statements;" >/dev/null 2>&1; then
+        echo " FAIL (pg_stat_statements not queryable)"; exit 1
+    fi
+    echo " OK"
+done
+
+# 3. testuser TCP password login against every node.
+for PORT in "${PRIMARY_PORT}" "${REPLICA1_PORT}" "${REPLICA2_PORT}"; do
+    echo -n "  - node ${PORT}: testuser TCP password login..."
+    if ! docker exec -e PGPASSWORD=testuser "${CONTAINER}" \
+         psql -h 127.0.0.1 -p "${PORT}" -U testuser -d testuser -tAc "SELECT 1" >/dev/null 2>&1; then
+        echo " FAIL (testuser could not authenticate)"; exit 1
+    fi
+    echo " OK"
+done
+
+printf "[%s] PgSQL replication verification COMPLETE (f/t/t + pg_stat_statements + testuser OK)\n" "$(date)"

test/infra/infra-dbdeployer-pgsql17-repl/bin/docker-proxy-post.bash

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Configure ProxySQL for the infra-dbdeployer-pgsql17-repl backend (automatic
+# rw-split via Toxiproxy + monitor-driven pg_is_in_recovery() demotion).
+#
+# Follows the infra-pgsql17-repl pattern: eval-expand the SQL template (so
+# ${INFRA_ID}/${WHG}/${RHG}/${ROOT_PASSWORD} are substituted) and pipe it into
+# the ProxySQL admin interface over the PG protocol (port 6132), NOT the MySQL
+# admin protocol -- ProxySQL's pgsql_* admin tables are only writable there.
+set -e
+set -o pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+[ -f "${SCRIPT_DIR}/../.env" ] && . "${SCRIPT_DIR}/../.env"
+PROXY_CONTAINER="proxysql.${INFRA_ID}"
+
+# ROOT_PASSWORD is normally exported by the caller (docker-compose-init.bash /
+# start-proxysql-isolated.bash), but ensure-infras.bash's "already running"
+# reconfigure path invokes this script directly without it. Re-derive it with
+# the same deterministic formula so the 'postgres' pgsql_users row keeps
+# matching the password the entrypoint actually set on the role.
+ROOT_PASSWORD="${ROOT_PASSWORD:-$(echo -n "${INFRA_ID}" | sha256sum | head -c 10)}"
+
+echo ">>> Configuring ProxySQL (${PROXY_CONTAINER}) for PGSQL Replication (automatic rw-split via Toxiproxy): ${INFRA}"
+
+# Wait for ProxySQL admin (MySQL protocol, port 6032) to be reachable.
+while ! docker exec "${PROXY_CONTAINER}" mysql -uadmin -padmin -h127.0.0.1 -P6032 -e 'SELECT 1' >/dev/null 2>&1; do
+    echo -n '.'
+    sleep 1
+done
+
+# Pre-process the SQL template.
+SQL_TEMPLATE=$(cat ./conf/proxysql/infra-config.sql)
+SQL_CONTENT=$(eval "echo \"${SQL_TEMPLATE}\"")
+
+# Apply configuration via docker exec using psql (ProxySQL Admin supports PG
+# protocol on port 6132). ON_ERROR_STOP=1 makes psql abort with a non-zero
+# exit on the FIRST SQL-level error (bad token, constraint violation, ...);
+# without it psql prints the error, keeps going, and exits 0 -- silently
+# defeating set -e and this script's fail-non-zero contract.
+echo "${SQL_CONTENT}" | docker exec -i "${PROXY_CONTAINER}" env PGPASSWORD='admin' psql -v ON_ERROR_STOP=1 -h127.0.0.1 -p6132 -Uadmin -dadmin

test/infra/infra-dbdeployer-pgsql17-repl/bin/toxiproxy-bootstrap.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Create one passthrough Toxiproxy proxy per PG backend (primary/replica1/
+# replica2). No toxics are added here -- SP-4's chaos suite adds those later.
+#
+# Idempotent: safe to re-run against an already-bootstrapped toxiproxy (each
+# proxy is deleted first, 404-on-delete is tolerated, and a 409 on create is
+# treated as "already exists" -> success).
+#
+# The toxiproxy:2.9.0 image ships no shell/curl, so the admin HTTP API (port
+# 8474) is driven from a throwaway curl container attached to the same
+# Docker network as toxiproxy and the backend.
+set -euo pipefail
+
+: "${INFRA_ID:?INFRA_ID must be set}"
+
+NETWORK="${INFRA_ID}_backend"
+TOXI_HOST="toxiproxy.${INFRA_ID}"
+TOXI_ADMIN="${TOXI_HOST}:8474"
+UPSTREAM_HOST="dbdeployer1.${INFRA_ID}"
+CURL_IMAGE="curlimages/curl:8.10.1"
+
+curl_in_net() {
+    docker run --rm --network "${NETWORK}" "${CURL_IMAGE}" "$@"
+}
+
+echo ">>> toxiproxy-bootstrap: waiting for Toxiproxy admin API at ${TOXI_ADMIN}..."
+MAX_WAIT=60
+COUNT=0
+until curl_in_net -fsS -o /dev/null "http://${TOXI_ADMIN}/version"; do
+    if [ "${COUNT}" -ge "${MAX_WAIT}" ]; then
+        echo "ERROR: Toxiproxy admin API at ${TOXI_ADMIN} did not become reachable within ${MAX_WAIT}s."
+        exit 1
+    fi
+    echo -n "."
+    sleep 2
+    COUNT=$((COUNT + 2))
+done
+echo " OK"
+
+mk() {  # name  listen_port  upstream_port
+    local name="$1" port="$2" upstream_port="$3"
+    local body="{\"name\":\"${name}\",\"listen\":\"0.0.0.0:${port}\",\"upstream\":\"${UPSTREAM_HOST}:${upstream_port}\",\"enabled\":true}"
+
+    echo -n "  - proxy ${name} (0.0.0.0:${port} -> ${UPSTREAM_HOST}:${upstream_port})..."
+
+    # Delete any pre-existing proxy of the same name; tolerate 404 (doesn't exist yet).
+    curl_in_net -sS -o /dev/null -XDELETE "http://${TOXI_ADMIN}/proxies/${name}" || true
+
+    local http_code
+    http_code=$(curl_in_net -sS -o /tmp/toxi_create_resp -w '%{http_code}' \
+        -XPOST "http://${TOXI_ADMIN}/proxies" -d "${body}" 2>/dev/null || echo "000")
+
+    if [ "${http_code}" = "200" ] || [ "${http_code}" = "201" ] || [ "${http_code}" = "409" ]; then
+        echo " OK (${http_code})"
+    else
+        echo " FAIL (HTTP ${http_code})"
+        exit 1
+    fi
+}
+
+mk pg_primary  6001 16710
+mk pg_replica1 6002 16711
+mk pg_replica2 6003 16712
+
+echo ">>> toxiproxy-bootstrap: verifying proxy list..."
+curl_in_net -fsS "http://${TOXI_ADMIN}/proxies"
+echo
+echo ">>> toxiproxy-bootstrap: done."