ci: Add Codecov coverage for cluster simulator groups

.github/workflows/CI-cluster-simulator.yml

@@ -24,7 +24,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  BUILD_CACHE_KEY: cluster-simulator-v3-ubuntu22-${{ github.sha }}
+  BUILD_CACHE_KEY: cluster-simulator-v3-ubuntu24-${{ github.sha }}
   RUNTIME_CACHE_DIR: .cluster-simulator-runtime
 
 jobs:
@@ -80,6 +80,9 @@ jobs:
     name: test / ${{ matrix.group }}
     needs: build
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write
     timeout-minutes: 120
     strategy:
       fail-fast: false
@@ -88,6 +91,7 @@ jobs:
     env:
       INFRA_ID: ${{ matrix.group }}-${{ github.run_id }}-${{ github.run_attempt }}
       TAP_GROUP: ${{ matrix.group }}
+      COVERAGE: "1"
 
     steps:
     - name: Checkout
@@ -125,6 +129,29 @@ jobs:
         test/infra/control/stop-proxysql-isolated.bash || true
         test/infra/control/destroy-infras.bash || true
 
+    - name: Require non-empty simulator coverage LCOV
+      # `disable_search` plus a missing explicit report would make Codecov a
+      # silent no-op. Treat coverage-generation regressions as CI failures,
+      # while keeping Codecov SaaS outages non-blocking below.
+      id: require-lcov
+      if: always()
+      run: test -s ci_infra_logs/${{ env.INFRA_ID }}/coverage-report/${{ env.INFRA_ID }}.info
+
+    - name: Upload coverage to Codecov
+      if: always() && steps.require-lcov.outcome == 'success'
+      uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # Codecov Action v4
+      with:
+        codecov_yml_path: codecov.yml
+        override_commit: ${{ github.event.pull_request.head.sha || github.sha }}
+        files: ci_infra_logs/${{ env.INFRA_ID }}/coverage-report/${{ env.INFRA_ID }}.info
+        flags: simulation-tests
+        name: ${{ matrix.group }}-coverage
+        use_oidc: true
+        disable_search: true
+        plugins: noop
+        fail_ci_if_error: false
+        verbose: true
+
     - name: Archive failure logs
       if: ${{ failure() && !cancelled() }}
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

.github/workflows/CI-lint-groups-json.yml

@@ -25,6 +25,8 @@ jobs:
         run: python3 test/tap/groups/lint_groups_json.py
       - name: Check every TAP source is registered in groups.json
         run: python3 test/tap/groups/check_groups.py --source
+      - name: Check cluster simulator coverage contract
+        run: test/infra/control/test-cluster-simulator-coverage.bash
       - name: Check coverage collector invariants
         run: test/infra/control/validate-coverage-gcov-toolchain.bash
       - name: Check group infra/workflow coverage (warn-only)

docs/superpowers/plans/2026-08-14-gcov-collector-compatibility.md

@@ -0,0 +1,213 @@
+# GCOV Collector Compatibility Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Make TAP coverage conversion use GCOV 11, matching the Ubuntu 22 coverage build, so Codecov receives daemon-side FFTO coverage.
+
+**Architecture:** Keep ProxySQL's GCC 11 coverage build unchanged. The Ubuntu 24 CI-base image gains `gcov-11`; the standalone and multi-group raw-GCOV conversion paths select it explicitly with fastcov's `-g` option and fail instead of silently writing an empty report when it is unavailable.
+
+**Tech Stack:** Docker, Ubuntu APT, Bash, fastcov, GCOV, GitHub Actions test infrastructure.
+
+## Global Constraints
+
+- Coverage build compiler remains GCC/GCOV 11.4.
+- Raw `.gcda` conversion must invoke `fastcov -g gcov-11`.
+- LCOV-only combination must remain toolchain-agnostic.
+- A missing compatible GCOV binary is a CI failure, not a warning.
+- Verify using real MySQL FFTO TAP traffic, not a synthetic packet helper.
+
+---
+
+### Task 1: Add a regression check for the coverage-toolchain contract
+
+**Files:**
+- Create: `test/infra/control/validate-coverage-gcov-toolchain.bash`
+- Test: `test/infra/control/validate-coverage-gcov-toolchain.bash`
+
+**Interfaces:**
+- Consumes: `test/infra/docker-base/Dockerfile`, `test/infra/control/run-tests-isolated.bash`, and `test/infra/control/run-multi-group.bash`.
+- Produces: exit status 0 only when the image installs `gcov-11` and both raw-data fastcov call sites specify `-g gcov-11`.
+
+- [ ] **Step 1: Write the failing regression check**
+
+Create an executable Bash script that resolves the repository root from its own path and checks these exact conditions:
+
+```bash
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+dockerfile="${root}/test/infra/docker-base/Dockerfile"
+runner="${root}/test/infra/control/run-tests-isolated.bash"
+multi="${root}/test/infra/control/run-multi-group.bash"
+
+rg -q '^[[:space:]]*gcov-11[[:space:]\\]*$' "${dockerfile}"
+test "$(rg -c 'fastcov -b .* -g gcov-11' "${runner}")" -eq 1
+test "$(rg -c 'fastcov -b .* -g gcov-11' "${multi}")" -eq 1
+```
+
+- [ ] **Step 2: Run the check and verify it fails**
+
+Run:
+
+```bash
+bash test/infra/control/validate-coverage-gcov-toolchain.bash
+```
+
+Expected: nonzero exit because `gcov-11` is absent and neither conversion command selects it.
+
+- [ ] **Step 3: Commit the failing-test addition**
+
+```bash
+git add test/infra/control/validate-coverage-gcov-toolchain.bash
+git commit -m "test(ci): assert GCOV collector compatibility"
+```
+
+### Task 2: Use the compiler-matching GCOV for raw coverage conversion
+
+**Files:**
+- Modify: `test/infra/docker-base/Dockerfile:8-33`
+- Modify: `test/infra/control/run-tests-isolated.bash:391-400`
+- Modify: `test/infra/control/run-multi-group.bash:361-378`
+- Test: `test/infra/control/validate-coverage-gcov-toolchain.bash`
+
+**Interfaces:**
+- Consumes: `gcov-11` supplied by the CI-base Docker image.
+- Produces: `fastcov` LCOV conversion from GCC 11 `.gcda` data, or an immediate nonzero CI exit with an explicit missing-tool message.
+
+- [ ] **Step 1: Install the matching reader in CI-base**
+
+Add `gcov-11` to the `apt-get install` list in `test/infra/docker-base/Dockerfile`, adjacent to `lcov`:
+
+```dockerfile
+    lcov \\
+    gcov-11 \\
+    && pip3 install --break-system-packages fastcov \\
+```
+
+- [ ] **Step 2: Make standalone conversion fail clearly and use GCOV 11**
+
+Immediately before the standalone `fastcov` invocation in the exit trap, add:
+
+```bash
+if ! command -v gcov-11 >/dev/null 2>&1; then
+    echo ">>> ERROR: gcov-11 is required to decode GCC 11 coverage data" >&2
+    exit 1
+fi
+```
+
+Then change the command to:
+
+```bash
+fastcov -b -g gcov-11 -j$(nproc) -l \\
+```
+
+Keep the existing source exclusions and report path unchanged.
+
+- [ ] **Step 3: Make multi-group conversion fail clearly and use GCOV 11**
+
+Inside the per-group `docker run ... bash -c` block, replace the conditional fastcov wrapper with:
+
+```bash
+set -e
+command -v gcov-11 >/dev/null 2>&1 || {
+    echo ">>> ERROR: gcov-11 is required to decode GCC 11 coverage data" >&2
+    exit 1
+}
+cd "${GCOV_DIR}"
+fastcov -b -g gcov-11 -j4 -l \\
+    -e /usr deps \\
+    -d . -o "${GROUP_INFO}" >> "${COVERAGE_LOG}" 2>&1
+```
+
+Remove the outer warning-only fallback for that raw-data conversion command so the coverage workflow cannot continue after an empty per-group report.
+
+- [ ] **Step 4: Run the regression check and verify it passes**
+
+Run:
+
+```bash
+bash test/infra/control/validate-coverage-gcov-toolchain.bash
+```
+
+Expected: exit 0.
+
+- [ ] **Step 5: Build and inspect the CI-base image**
+
+Run:
+
+```bash
+docker build -t proxysql-ci-base:gcov11 test/infra/docker-base
+docker run --rm proxysql-ci-base:gcov11 bash -lc 'gcov-11 --version && fastcov --help | grep -F -- "-g GCOV"'
+```
+
+Expected: GCOV 11 is available and fastcov supports selecting a GCOV executable.
+
+- [ ] **Step 6: Commit the compatibility fix**
+
+```bash
+git add test/infra/docker-base/Dockerfile test/infra/control/run-tests-isolated.bash test/infra/control/run-multi-group.bash test/infra/control/validate-coverage-gcov-toolchain.bash
+git commit -m "fix(ci): use matching GCOV for TAP coverage"
+```
+
+### Task 3: Prove end-to-end coverage with real FFTO traffic
+
+**Files:**
+- Test: existing `test/tap/tests/test_ffto_mysql-t.cpp`
+- Test: generated `ci_infra_logs/<infra-id>/coverage-report/<infra-id>.info`
+
+**Interfaces:**
+- Consumes: the rebuilt `proxysql-ci-base:latest`, GCC 11 ProxySQL coverage binary, and isolated legacy MySQL infrastructure.
+- Produces: a passing TAP test and nonzero LCOV lines for `lib/MySQLFFTO.cpp`.
+
+- [ ] **Step 1: Rebuild/tag the fixed CI-base image for the isolated scripts**
+
+```bash
+docker build -t proxysql-ci-base:latest test/infra/docker-base
+```
+
+- [ ] **Step 2: Build the existing GCC 11 coverage binary and the one TAP executable**
+
+```bash
+WITHGCOV=1 PROXYSQL40=1 make ubuntu22-tap-genai-gcov
+docker run --rm -v "$PWD:/opt/proxysql" proxysql/packaging:build-ubuntu22-v4.0.0 \\
+  bash -lc 'cd /opt/proxysql/test/tap/tests && WITHGCOV=1 PROXYSQL40=1 PROXYSQL31=1 PROXYSQLFFTO=1 make test_ffto_mysql-t'
+```
+
+- [ ] **Step 3: Run only real FFTO TAP traffic under coverage**
+
+```bash
+export INFRA_ID=gcov11-ffto
+export TAP_GROUP=legacy-g4
+export INFRA_TYPE=infra-mysql57
+export COVERAGE=1
+test/infra/control/ensure-infras.bash
+TEST_PY_TAP_INCL=test_ffto_mysql-t test/infra/control/run-tests-isolated.bash
+```
+
+Expected: `test_ffto_mysql-t` passes, including its fast-forward-session and query-digest assertions.
+
+- [ ] **Step 4: Assert that LCOV records FFTO execution**
+
+```bash
+info="ci_infra_logs/${INFRA_ID}/coverage-report/${INFRA_ID}.info"
+awk '/^SF:.*lib\/MySQLFFTO\.cpp$/{seen=1} seen && /^LH:/{print; exit}' "${info}"
+test -s "${info}"
+```
+
+Expected: a nonzero `LH:` value for `lib/MySQLFFTO.cpp`.
+
+- [ ] **Step 5: Tear down only the isolated test resources**
+
+```bash
+INFRA_ID="${INFRA_ID}" TAP_GROUP=legacy-g4 INFRA_TYPE=infra-mysql57 \\
+  test/infra/control/destroy-infras.bash
+```
+
+- [ ] **Step 6: Commit any verification-only script changes, if created**
+
+```bash
+git status --short
+```
+
+Expected: no generated coverage artifacts or infrastructure logs are staged or committed.

docs/superpowers/specs/2026-08-14-gcov-collector-compatibility-design.md

@@ -0,0 +1,29 @@
+# GCOV collector compatibility
+
+## Problem
+
+The coverage build runs with GCC 11.4, so its `.gcno` and `.gcda` files must
+be read by the matching GCOV 11 tool. The Ubuntu 24 CI-base image currently
+provides GCOV 13 only. Fastcov therefore discovers the data files but cannot
+convert them, producing an empty LCOV report while the TAP tests themselves
+pass and exercise the implementation.
+
+## Design
+
+Keep the coverage build toolchain unchanged. Install `gcov-11` in the CI-base
+image and use it explicitly for every fastcov invocation that converts raw
+GCOV files. The two conversion paths are the standalone test-runner exit trap
+and the multi-group per-group collector. Combining existing LCOV files does
+not need GCOV and remains unchanged.
+
+Before conversion, each path will check that `gcov-11` is available and fail
+with an actionable error if it is not. This prevents a successful TAP job from
+silently uploading an empty coverage report.
+
+## Verification
+
+Add a regression check for the collector contract: the CI-base Dockerfile
+must install `gcov-11`, and raw-data fastcov calls must use `-g gcov-11`.
+Then run the existing `test_ffto_mysql-t` through the normal isolated runner
+with coverage enabled. The test must pass and its LCOV output must include
+nonzero coverage for `lib/MySQLFFTO.cpp`.

test/infra/control/cluster-simulator-ci.bash

@@ -189,12 +189,12 @@ handle_build() {
             --env "GIT_VERSION_BASE=${git_version}" \
             --entrypoint /opt/proxysql/test/infra/control/cluster-simulator-ci.bash \
             --workdir /opt/proxysql \
-            ubuntu22_build _build
+            ubuntu24_build _build
     )
 }
 
 # _build
-# Purpose: Execute the compiler commands inside the Ubuntu 22 packaging image.
+# Purpose: Execute the compiler commands inside the Ubuntu 24 packaging image.
 # Local use: Internal only; use the public `build` command from the host.
 # GitHub use: Called by `build` as the packaging container entrypoint.
 handle_internal_build() {
@@ -204,11 +204,14 @@ handle_internal_build() {
         die "GIT_VERSION_BASE was not provided by the host build command."
 
     cd "${REPO_ROOT}"
-    make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" testall
-    make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" build_cluster_simulator
-    make -C test/deps/cluster_simulator -j"$(nproc)" check
-    make -C test/tap -j"$(nproc)" GIT_VERSION="${GIT_VERSION_BASE}" tap
-    make -C test/tap/tests -j"$(nproc)" \
+    make -j"$(nproc)" WITHGCOV=1 \
+        GIT_VERSION_BASE="${GIT_VERSION_BASE}" testall
+    make -j"$(nproc)" WITHGCOV=1 \
+        GIT_VERSION_BASE="${GIT_VERSION_BASE}" build_cluster_simulator
+    make -C test/deps/cluster_simulator -j"$(nproc)" WITHGCOV=1 check
+    make -C test/tap -j"$(nproc)" WITHGCOV=1 \
+        GIT_VERSION="${GIT_VERSION_BASE}" tap
+    make -C test/tap/tests -j"$(nproc)" WITHGCOV=1 \
         GIT_VERSION="${GIT_VERSION_BASE}" "${SIMULATOR_BINARIES[@]}"
 }
 
@@ -258,6 +261,13 @@ handle_stage() {
     install -D -m 0755 \
         "${REPO_ROOT}/src/proxysql" \
         "${STAGE_TEMP_DIR}/src/proxysql"
+    install -d \
+        "${STAGE_TEMP_DIR}/lib/obj" \
+        "${STAGE_TEMP_DIR}/src/obj"
+    cp -a "${REPO_ROOT}"/lib/obj/*.gcno \
+        "${STAGE_TEMP_DIR}/lib/obj/"
+    cp -a "${REPO_ROOT}"/src/obj/*.gcno \
+        "${STAGE_TEMP_DIR}/src/obj/"
     install -D -m 0755 \
         "${REPO_ROOT}/test/deps/cluster_simulator/cluster_simulator" \
         "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/cluster_simulator"

test/infra/control/test-cluster-simulator-coverage.bash

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+runner="${root}/test/infra/control/run-tests-isolated.bash"
+builder="${root}/test/infra/control/cluster-simulator-ci.bash"
+workflow="${root}/.github/workflows/CI-cluster-simulator.yml"
+dump_helper="${root}/test/infra/control/dump-proxysql-gcov.bash"
+status_helper="${root}/test/infra/control/coverage-exit-status.bash"
+
+test -x "${dump_helper}"
+test -f "${status_helper}"
+
+source "${status_helper}"
+coverage_exit_status 0 0
+if coverage_exit_status 0 1; then
+    echo "coverage failure must fail a successful TAP run" >&2
+    exit 1
+fi
+if coverage_exit_status 7 0; then
+    echo "TAP failure must preserve its original exit status" >&2
+    exit 1
+fi
+
+grep -Fq 'make -C test/deps/cluster_simulator -j"$(nproc)" WITHGCOV=1 check' "${builder}"
+dump_line=$(grep -nF 'dump-proxysql-gcov.bash' "${runner}" | cut -d: -f1)
+decode_line=$(grep -nF 'fastcov -b' "${runner}" | head -n1 | cut -d: -f1)
+test "${dump_line}" -lt "${decode_line}"
+grep -Fq 'id: require-lcov' "${workflow}"
+grep -Fq 'run: test -s ci_infra_logs/${{ env.INFRA_ID }}/coverage-report/${{ env.INFRA_ID }}.info' "${workflow}"
+grep -Fq "if: always() && steps.require-lcov.outcome == 'success'" "${workflow}"
+
+echo "cluster simulator coverage contract passed"