diff --git a/.github/workflows/CI-lint-groups-json.yml b/.github/workflows/CI-lint-groups-json.yml index 07898565b6..9981f3cd42 100644 --- a/.github/workflows/CI-lint-groups-json.yml +++ b/.github/workflows/CI-lint-groups-json.yml @@ -10,13 +10,24 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Fetch GH-Actions branch + # The reusable half of every workflow pair lives on GH-Actions, so the + # coverage lint needs that ref to tell an unwired group from one wired + # only over there. actions/checkout fetches a single branch, so without + # this the check finds no ref and skips itself silently. --depth=1 is + # enough: the lint only reads the tree, never history. + run: | + git fetch --no-tags --depth=1 origin \ + +refs/heads/GH-Actions:refs/remotes/origin/GH-Actions \ + || echo "::warning::could not fetch GH-Actions; the workflow-coverage check will skip" - name: Lint groups.json format run: python3 test/tap/groups/lint_groups_json.py - name: Check every TAP source is registered in groups.json run: python3 test/tap/groups/check_groups.py --source - name: Check group infra/workflow coverage (warn-only) - # Warns when a group references a missing infra (phantom) or has a - # real dbdeployer infra but no GitHub Actions workflow. Known gaps - # are allowlisted; only NEW infra-backed groups without a workflow - # are flagged. Warn-only here (no --strict) so it never reds CI. + # Warns when a group references a missing infra (phantom), or when no + # workflow on either branch can select the group at all -- meaning + # tests registered in it never run in CI. Known gaps are allowlisted; + # only NEW ones are flagged. Warn-only here (no --strict) so it never + # reds CI. run: python3 test/tap/groups/lint_group_coverage.py diff --git a/test/tap/groups/lint_group_coverage.py b/test/tap/groups/lint_group_coverage.py index 37bc482cd7..401455a5eb 100755 --- a/test/tap/groups/lint_group_coverage.py +++ b/test/tap/groups/lint_group_coverage.py @@ -9,9 +9,16 @@ (a "phantom" group that cannot start any backend, e.g. the old mysql91 / mysql92 stubs, gr and non-gr) - B. Does a group that DOES have a real dbdeployer infra lack a GitHub - Actions caller workflow (.github/workflows/CI-.yml), i.e. it - can run locally via run-tests-isolated.bash but never runs in CI? + B. Can any GitHub Actions workflow actually select the group, i.e. does + it run in CI at all -- or only locally via run-tests-isolated.bash? + + A group counts as wired if a CI-.yml caller exists, OR its name + appears in any workflow on this branch or on origin/GH-Actions (where + the reusable half of every pair lives), OR it belongs to a dynamically + discovered family. Groups with no infras.lst are checked too: needing no + backend does not mean needing no workflow. Checking only for a + CI-.yml filename on this branch is what let no-infra-g1 sit + unwired since it was created. Group -> infra resolution mirrors ensure-infras.bash: BASE_GROUP = with a trailing -g/_g stripped; infra names @@ -22,7 +29,7 @@ Severity: WARN-ONLY by default -- this never reds CI on its own. Known, tracked coverage gaps live in ALLOWLIST_NO_WORKFLOW so only *newly* -introduced infra-backed groups without a workflow are called out as NEW. +introduced groups without a workflow are called out as NEW. Pass --strict to turn phantom-infra and NEW missing-workflow findings into a non-zero exit (for a future opt-in enforcement step). @@ -34,6 +41,7 @@ import json import os import re +import subprocess import sys SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -55,6 +63,24 @@ "mysql95-binlog", "mysqlx-soak", "pgsql17-repl", + # --- pre-existing debt surfaced when the check was widened ------------- + # These were invisible while the linter only inspected infra-backed groups + # and only looked for a CI-.yml filename. They are unwired on both + # branches, i.e. tests registered in them do not run in CI. Allowlisted so + # the linter reports only genuinely NEW gaps; trim as workflows land. + "mysql-auto_increment_delay_multiplex=0", + "mysql-multiplexing=false", + "mysql-query_digests=0", + "mysql-query_digests_keep_comment=1", + "mysql91-gr", + "mysql92-gr", + "mysqlx-e2e", + "pgsql-repl", + "todo", + # NOTE: 'no-infra' is deliberately NOT allowlisted. It is a real, currently + # unwired group (5 tests, incl. reg_test_5363_admin_monitor_caching_sha2-t) + # and finding it is what prompted widening this check. Add CI-no-infra-g1.yml + # (+ ci-no-infra-g1.yml@GH-Actions) rather than silencing it here. } @@ -90,20 +116,101 @@ def read_infra_names(group): return None -def workflow_exists(group): +# Group families whose CI wiring is generated at run time rather than written +# out as a CI-.yml, so a static name lookup cannot see them. +# cluster_sim_* -> CI-cluster-simulator.yml builds its matrix from +# `test/infra/control/cluster-simulator-ci.bash discover`, +# which selects every groups.json entry starting with this +# prefix. Adding such a group wires it up automatically. +DYNAMIC_DISCOVERY_PREFIXES = ("cluster_sim_",) + + +def _local_workflow_blob(): + """Filenames + contents of .github/workflows on the current branch.""" + parts = [] + wf_dir = os.path.join(REPO_ROOT, ".github", "workflows") + for root, _dirs, files in os.walk(wf_dir): + for name in files: + parts.append(name) + try: + with open(os.path.join(root, name), encoding="utf-8", errors="replace") as f: + parts.append(f.read()) + except OSError: + pass + return "\n".join(parts) + + +def _gh_actions_workflow_blob(): + """Filenames + contents of .github/workflows on origin/GH-Actions. + + Returns None when the ref is unavailable (shallow clone, no remote, no git). + The caller must then skip the workflow check rather than report false gaps: + the reusable half of every workflow pair lives on that branch, so without it + we cannot tell a genuinely unwired group from one wired only over there. + """ + try: + listing = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", "origin/GH-Actions", ".github/workflows"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False, + ) + if listing.returncode != 0: + return None + files = listing.stdout.split() + if not files: + return None + parts = ["\n".join(files)] + for path in files: + blob = subprocess.run( + ["git", "show", f"origin/GH-Actions:{path}"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False, + ) + if blob.returncode == 0: + parts.append(blob.stdout) + return "\n".join(parts) + except (OSError, subprocess.SubprocessError): + return None + + +def workflow_covers(group, local_blob, gh_blob): + """True when this group can actually be selected by some CI workflow. + + A group is wired up if ANY of these hold: + 1. a caller file is named after it (.github/workflows/CI-.yml) + 2. its name appears anywhere in a workflow on this branch -- covers + groups selected by an env/matrix entry rather than a dedicated file, + e.g. 'TAP_GROUP: mysqlx-tsan-g1' inside a larger workflow + 3. its name appears in a workflow on origin/GH-Actions (the reusable half) + 4. it belongs to a family discovered dynamically (see + DYNAMIC_DISCOVERY_PREFIXES) + + Checking only (1) is what let no-infra-g1 go unnoticed in both directions: + it has no dedicated caller, and matching names in file *contents* is needed + to avoid flagging the matrix-driven groups that are genuinely wired. + """ + if group.startswith(DYNAMIC_DISCOVERY_PREFIXES): + return True for ext in ("yml", "yaml"): - path = os.path.join(REPO_ROOT, ".github", "workflows", f"CI-{group}.{ext}") - if os.path.isfile(path): + if os.path.isfile(os.path.join(REPO_ROOT, ".github", "workflows", f"CI-{group}.{ext}")): return True + if group in local_blob: + return True + if gh_blob is not None and group in gh_blob: + return True return False -def classify_group(group): +def classify_group(group, local_blob, gh_blob, check_workflows=True): """Return (missing_infras, workflow_state) for one group. missing_infras: list of infra names referenced but absent on disk. - workflow_state: None if not infra-backed or a workflow exists; - "new" / "known" when a real infra has no workflow. + workflow_state: None when a workflow covers the group, the group is a + phantom, or workflow checking is disabled; + "new" / "known" when nothing in CI can select the group. + + NOTE: an absent or empty infras.lst does NOT exempt a group. Such a group + still needs a workflow to ever run in CI -- it simply needs no backend. + Exempting them is exactly why no-infra-g1 was never reported despite having + no CI wiring on either branch since it was created. """ concrete = read_infra_names(group) or [] missing = [ @@ -111,7 +218,9 @@ def classify_group(group): if not os.path.isdir(os.path.join(REPO_ROOT, "test", "infra", n)) ] workflow_state = None - if concrete and not missing and not workflow_exists(group): + # A phantom group cannot start its backend, so demanding a workflow for it + # would just be noise on top of the phantom finding. + if check_workflows and not missing and not workflow_covers(group, local_blob, gh_blob): workflow_state = "known" if base_group(group) in ALLOWLIST_NO_WORKFLOW else "new" return missing, workflow_state @@ -139,11 +248,21 @@ def lint_coverage(groups_path, strict=False): with open(groups_path, encoding="utf-8") as f: data = json.load(f) + local_blob = _local_workflow_blob() + gh_blob = _gh_actions_workflow_blob() + check_workflows = gh_blob is not None + if not check_workflows: + print("NOTE origin/GH-Actions is not available (shallow clone or missing " + "remote); skipping the missing-workflow check. The reusable half of " + "every workflow pair lives on that branch, so without it any finding " + "would be a guess. Run `git fetch origin GH-Actions` to enable it.", + file=sys.stderr) + phantom = [] # (group, missing_infra) missing_wf_new = [] missing_wf_known = [] for group in sorted(collect_groups(data)): - missing, wf_state = classify_group(group) + missing, wf_state = classify_group(group, local_blob, gh_blob, check_workflows) phantom.extend((group, infra) for infra in missing) if wf_state == "new": missing_wf_new.append(group) @@ -154,14 +273,15 @@ def lint_coverage(groups_path, strict=False): print(f"WARN [phantom-infra] group '{group}': infras.lst references " f"missing infra 'test/infra/{infra}'", file=sys.stderr) for group in missing_wf_new: - print(f"WARN [missing-workflow:NEW] group '{group}' has a dbdeployer " - f"infra but no .github/workflows/CI-{group}.yml -- it will never " - f"run in GitHub Actions. Add the caller (+ ci-{group}.yml@GH-Actions) " - f"or, if intentional, add '{base_group(group)}' to ALLOWLIST_NO_WORKFLOW.", + print(f"WARN [missing-workflow:NEW] group '{group}' is not selectable by " + f"any workflow on this branch or on origin/GH-Actions -- tests " + f"registered in it never run in CI. Add the caller " + f".github/workflows/CI-{group}.yml (+ ci-{group}.yml@GH-Actions), or " + f"if intentional add '{base_group(group)}' to ALLOWLIST_NO_WORKFLOW.", file=sys.stderr) for group in missing_wf_known: - print(f"note [missing-workflow:known] group '{group}' has infra but no " - f"workflow (allowlisted family '{base_group(group)}')") + print(f"note [missing-workflow:known] group '{group}' has no workflow " + f"(allowlisted family '{base_group(group)}')") print( f"\ngroup coverage lint: {len(collect_groups(data))} groups | "