From ceb5a2d76fec54cd9f788a3468b4f07ddc31789b Mon Sep 17 00:00:00 2001 From: Viktor Didkovskyi Date: Wed, 12 Aug 2026 12:30:14 +0300 Subject: [PATCH 1/3] test: cover the gate detector so the baseline cannot regress silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector had no committed tests. It was verified once by hand against the organization's workflows, which proves nothing about the next edit. Add 29 tests that extract the audit script out of the workflow YAML and execute it, rather than re-implementing the logic. The code under test is therefore the code that ships, and a change to the workflow changes what the tests exercise. Coverage is split by consequence. Fake gates that must be blocked: inline and block-scalar echo, a set -e prefix, any other phrase, exit 0, true, chained inert commands, and a checkout step followed by an echo — the shape actually present in meditation-service and healify-org. Real gates that must pass, which matter more because a false positive here red-lines a compliant repository: compound commands after an echo, command words that merely start with an inert word, semicolon and pipe separation, an exit 1 guard, aggregation through needs..result, and a gate sharing a file with an unrelated job that mentions the phrase. Advisory behaviour is pinned in both directions: it must annotate without failing by default and must fail under strict, each advisory class must be detected, and SHA-pinned and local actions must not be reported as mutable. Robustness covers unparsable YAML, an empty workflow directory, the step summary, and a clean stderr. Run them in CI through a separate workflow. They cannot live in the reusable baseline, which executes against the caller's checkout and so cannot see this repository's test files. --- .github/tests/test_baseline_audit.py | 291 ++++++++++++++++++++++++ .github/workflows/baseline-selftest.yml | 49 ++++ 2 files changed, 340 insertions(+) create mode 100644 .github/tests/test_baseline_audit.py create mode 100644 .github/workflows/baseline-selftest.yml diff --git a/.github/tests/test_baseline_audit.py b/.github/tests/test_baseline_audit.py new file mode 100644 index 0000000..7a3ae2c --- /dev/null +++ b/.github/tests/test_baseline_audit.py @@ -0,0 +1,291 @@ +"""Tests for the gate detector in organization-pr-baseline.yml. + +The audit runs as an inline `shell: python` step so the reusable workflow +carries no dependency on files in this repository. That makes it easy for a +test to drift from what actually ships, so these tests do not re-implement the +logic: they extract the exact script out of the workflow YAML and execute it. +Change the workflow and these tests exercise the change. + +Run: python3 .github/tests/test_baseline_audit.py +""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / ".github/workflows/organization-pr-baseline.yml" +AUDIT_STEP = "Audit workflow structure" + + +def audit_source() -> str: + """The exact script shipped in the workflow, not a copy of it.""" + doc = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + for step in doc["jobs"]["validate"]["steps"]: + if step.get("name") == AUDIT_STEP: + return step["run"] + raise AssertionError(f"step {AUDIT_STEP!r} not found in {WORKFLOW}") + + +class AuditResult: + def __init__(self, exit_code, stdout, stderr, summary): + self.exit_code = exit_code + self.stdout = stdout + self.stderr = stderr + self.summary = summary + + @property + def blocking(self): + return [x for x in self.stdout.splitlines() if x.startswith("::error")] + + @property + def advisory(self): + return [x for x in self.stdout.splitlines() if x.startswith("::warning")] + + +def run_audit(workflows, strict=False): + """Execute the shipped audit against a synthetic .github/workflows tree.""" + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp, ".github", "workflows") + target.mkdir(parents=True) + for name, body in workflows.items(): + (target / name).write_text(body, encoding="utf-8") + summary = Path(tmp, "summary.md") + summary.touch() + env = { + **os.environ, + "STRICT": "true" if strict else "false", + "GITHUB_STEP_SUMMARY": str(summary), + } + proc = subprocess.run( + [sys.executable, "-c", audit_source()], + cwd=tmp, + env=env, + capture_output=True, + text=True, + ) + return AuditResult( + proc.returncode, + proc.stdout, + proc.stderr, + summary.read_text(encoding="utf-8"), + ) + + +def gate(steps, name="CI Summary", needs=""): + """A minimal workflow whose single job is a merge gate.""" + return ( + "name: t\n" + "on: [pull_request]\n" + "permissions:\n" + " contents: read\n" + "jobs:\n" + " s:\n" + f" name: {name}\n" + " runs-on: ubuntu-24.04\n" + " timeout-minutes: 5\n" + + (f" {needs}\n" if needs else "") + + " steps:\n" + + textwrap.indent(textwrap.dedent(steps), " " * 6) + ) + + +class FakeGatesAreBlocked(unittest.TestCase): + """A required gate that cannot fail must be reported.""" + + def assert_blocked(self, steps, msg): + result = run_audit({"ci.yml": gate(steps)}) + self.assertTrue(result.blocking, f"{msg}\nstdout:\n{result.stdout}") + self.assertEqual(result.exit_code, 1, msg) + + def test_inline_echo(self): + self.assert_blocked('- run: echo "CI checks passed"\n', "inline echo placeholder") + + def test_block_scalar_echo(self): + self.assert_blocked('- run: |\n echo "CI checks passed"\n', "block scalar echo") + + def test_set_e_then_echo(self): + # Defeated the previous literal-string matcher. + self.assert_blocked('- run: |\n set -e\n echo "CI checks passed"\n', "set -e then echo") + + def test_any_other_phrase(self): + self.assert_blocked('- run: echo "all good"\n', "phrase-independent") + + def test_exit_zero(self): + self.assert_blocked("- run: exit 0\n", "exit 0") + + def test_true(self): + self.assert_blocked("- run: 'true'\n", "true") + + def test_chained_inert_commands(self): + self.assert_blocked("- run: echo a && echo b\n", "every segment inert") + + def test_multiline_all_inert(self): + self.assert_blocked("- run: |\n set -e\n true\n exit 0\n", "all lines inert") + + def test_setup_action_plus_echo(self): + # The shape found in meditation-service and healify-org: a checkout step + # does not make a gate meaningful. + self.assert_blocked( + '- uses: actions/checkout@v4\n- run: echo "CI checks passed"\n', + "setup action plus echo is still fake", + ) + + +class RealGatesArePermitted(unittest.TestCase): + """A false positive here red-lines a compliant repository, so these matter most.""" + + def assert_clean(self, steps, msg, needs=""): + result = run_audit({"ci.yml": gate(steps, needs=needs)}) + self.assertEqual(result.blocking, [], f"{msg}\nstdout:\n{result.stdout}") + self.assertEqual(result.exit_code, 0, msg) + + def test_real_command(self): + self.assert_clean("- run: npm test\n", "a real command is a real gate") + + def test_compound_command_after_echo(self): + # Prefix matching used to read this whole line as inert. + self.assert_clean('- run: echo "verifying" && make verify\n', "echo && make verify") + + def test_command_prefixed_by_inert_word(self): + self.assert_clean("- run: truncate_logs --check\n", "'truncate_logs' is not 'true'") + + def test_semicolon_separated(self): + self.assert_clean("- run: cd app; pytest\n", "cd then pytest") + + def test_piped_into_real_command(self): + self.assert_clean("- run: echo x | grep -q ok\n", "piped into grep") + + def test_exit_nonzero_guard(self): + self.assert_clean('- run: |\n if [ -z "$X" ]; then exit 1; fi\n', "exit 1 can fail") + + def test_aggregator_via_needs(self): + self.assert_clean( + '- run: |\n if [ "${{ needs.build.result }}" != "success" ]; then exit 1; fi\n', + "aggregates through needs", + needs="needs: [build]", + ) + + def test_unrelated_job_mentioning_the_phrase(self): + """The old matcher was file-scoped and fired on this.""" + body = ( + "name: t\n" + "on: [pull_request]\n" + "permissions:\n" + " contents: read\n" + "jobs:\n" + " other:\n" + " name: Other\n" + " runs-on: ubuntu-24.04\n" + " timeout-minutes: 5\n" + " steps:\n" + ' - run: echo "CI checks passed"\n' + " s:\n" + " name: CI Summary\n" + " runs-on: ubuntu-24.04\n" + " timeout-minutes: 5\n" + " steps:\n" + " - run: ./scripts/aggregate.sh\n" + ) + result = run_audit({"ci.yml": body}) + self.assertEqual(result.blocking, [], f"job-scoped, not file-scoped\n{result.stdout}") + + def test_non_gate_job_is_never_blocking(self): + result = run_audit({"ci.yml": gate('- run: echo "hi"\n', name="Build")}) + self.assertEqual(result.blocking, [], "only gate jobs are judged") + + +class AdvisoryTier(unittest.TestCase): + """Advisory findings annotate by default and fail only under strict.""" + + MUTABLE = ( + "name: t\n" + "on: [pull_request]\n" + "jobs:\n" + " b:\n" + " runs-on: ubuntu-24.04\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + ' - run: echo "${{ secrets.TOKEN }}"\n' + ) + + def test_advisory_does_not_fail_by_default(self): + result = run_audit({"ci.yml": self.MUTABLE}) + self.assertEqual(result.exit_code, 0, "advisory must not fail the default run") + self.assertTrue(result.advisory, "expected advisory findings") + + def test_advisory_fails_under_strict(self): + result = run_audit({"ci.yml": self.MUTABLE}, strict=True) + self.assertEqual(result.exit_code, 1, "strict must fail on advisory findings") + + def test_detects_each_advisory_class(self): + joined = "\n".join(run_audit({"ci.yml": self.MUTABLE}).advisory) + for expected in ( + "mutable ref", + "permissions block", + "timeout-minutes", + "interpolates a secret", + ): + self.assertIn(expected, joined, f"missing advisory: {expected}") + + def test_sha_pinned_action_is_not_flagged(self): + body = self.MUTABLE.replace( + "actions/checkout@v4", + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262", + ) + joined = "\n".join(run_audit({"ci.yml": body}).advisory) + self.assertNotIn("mutable ref", joined, "a full SHA must not be reported as mutable") + + def test_local_action_is_not_flagged(self): + body = self.MUTABLE.replace("actions/checkout@v4", "./.github/actions/setup") + joined = "\n".join(run_audit({"ci.yml": body}).advisory) + self.assertNotIn("mutable ref", joined, "local actions have no ref to pin") + + +class Robustness(unittest.TestCase): + def test_unparsable_workflow_is_blocking(self): + result = run_audit({"broken.yml": "jobs:\n - this: [is\n not: valid\n"}) + self.assertEqual(result.exit_code, 1) + self.assertTrue(any("Unable to parse" in x for x in result.blocking)) + + def test_empty_workflow_directory_is_clean(self): + result = run_audit({}) + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.blocking, []) + + def test_step_summary_is_written(self): + result = run_audit({"ci.yml": gate("- run: npm test\n")}) + self.assertIn("Organization PR Baseline", result.summary) + self.assertIn("Workflows audited", result.summary) + + def test_script_writes_nothing_to_stderr(self): + result = run_audit({"ci.yml": gate("- run: npm test\n")}) + self.assertEqual(result.stderr, "", f"unexpected stderr: {result.stderr}") + + +class ShippedWorkflowIsSelfConsistent(unittest.TestCase): + def test_audit_step_parses_as_python(self): + ast.parse(audit_source()) + + def test_this_repository_passes_its_own_blocking_tier(self): + workflows = { + p.name: p.read_text(encoding="utf-8") + for p in (REPO_ROOT / ".github/workflows").glob("*.y*ml") + } + result = run_audit(workflows) + self.assertEqual( + result.blocking, [], f"this repo must pass its own blocking tier\n{result.stdout}" + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/baseline-selftest.yml b/.github/workflows/baseline-selftest.yml new file mode 100644 index 0000000..c1fa47d --- /dev/null +++ b/.github/workflows/baseline-selftest.yml @@ -0,0 +1,49 @@ +name: Baseline Self-Test + +# Guards the organization PR baseline against regressions. The audit ships as an +# inline script inside organization-pr-baseline.yml, so these tests extract that +# script and run it — the code under test is provably the code that ships. +# +# This workflow is deliberately not part of the reusable baseline: it needs files +# from this repository, and a reusable workflow runs against the caller's checkout. + +on: + pull_request: + paths: + - ".github/workflows/organization-pr-baseline.yml" + - ".github/workflows/baseline-selftest.yml" + - ".github/tests/**" + push: + branches: [main] + paths: + - ".github/workflows/organization-pr-baseline.yml" + - ".github/tests/**" + +permissions: + contents: read + +concurrency: + group: baseline-selftest-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + name: Gate detector tests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out without retaining credentials + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install the YAML parser + run: python -m pip install --quiet --disable-pip-version-check pyyaml + + - name: Run gate detector tests + run: python .github/tests/test_baseline_audit.py From 0f16d0524ce437322ccca1d913459def02973d8d Mon Sep 17 00:00:00 2001 From: Viktor Didkovskyi Date: Wed, 12 Aug 2026 15:49:56 +0300 Subject: [PATCH 2/3] test: restrict the audit subprocess environment and bound its runtime Three fixes from review. The harness copied the full test-runner environment into a script that is read out of a workflow file, and a pull request can change that file. Pass only the two variables the script reads, plus PATH, so a modified audit step has nothing inherited to disclose. All 29 tests pass under the restricted environment, which confirms nothing else was needed. Bound the subprocess with a 60 second timeout so a detector regression that hangs fails the test pointing at the audit, rather than stalling until the job timeout. Align the push path filter with the pull_request one: a push to main touching only baseline-selftest.yml would previously have skipped the self-test. --- .github/tests/test_baseline_audit.py | 9 ++++++++- .github/workflows/baseline-selftest.yml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/tests/test_baseline_audit.py b/.github/tests/test_baseline_audit.py index 7a3ae2c..68b7e74 100644 --- a/.github/tests/test_baseline_audit.py +++ b/.github/tests/test_baseline_audit.py @@ -25,6 +25,9 @@ REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github/workflows/organization-pr-baseline.yml" AUDIT_STEP = "Audit workflow structure" +# A detector regression should fail fast and point at the subprocess, not +# hang until the CI job timeout. +AUDIT_TIMEOUT_SECONDS = 60 def audit_source() -> str: @@ -61,10 +64,13 @@ def run_audit(workflows, strict=False): (target / name).write_text(body, encoding="utf-8") summary = Path(tmp, "summary.md") summary.touch() + # Deliberately not os.environ: the script under test is read out of a + # workflow file that a pull request can modify, so it is given only the + # two variables it reads and nothing else to disclose. env = { - **os.environ, "STRICT": "true" if strict else "false", "GITHUB_STEP_SUMMARY": str(summary), + "PATH": os.environ.get("PATH", ""), } proc = subprocess.run( [sys.executable, "-c", audit_source()], @@ -72,6 +78,7 @@ def run_audit(workflows, strict=False): env=env, capture_output=True, text=True, + timeout=AUDIT_TIMEOUT_SECONDS, ) return AuditResult( proc.returncode, diff --git a/.github/workflows/baseline-selftest.yml b/.github/workflows/baseline-selftest.yml index c1fa47d..a143a4b 100644 --- a/.github/workflows/baseline-selftest.yml +++ b/.github/workflows/baseline-selftest.yml @@ -17,6 +17,7 @@ on: branches: [main] paths: - ".github/workflows/organization-pr-baseline.yml" + - ".github/workflows/baseline-selftest.yml" - ".github/tests/**" permissions: From ff0c0a17db3c9820d320cefc84bd225374a0ccd5 Mon Sep 17 00:00:00 2001 From: Viktor Didkovskyi Date: Wed, 12 Aug 2026 15:54:47 +0300 Subject: [PATCH 3/3] test: pass no inherited environment to the audit subprocess Keeping PATH was a half-measure. The script imports only stdlib and yaml and never shells out, so it needs no inherited value at all. It now receives exactly the two variables it reads. All 29 tests pass, which confirms it. Drop the now-unused os import. --- .github/tests/test_baseline_audit.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/tests/test_baseline_audit.py b/.github/tests/test_baseline_audit.py index 68b7e74..f0ac349 100644 --- a/.github/tests/test_baseline_audit.py +++ b/.github/tests/test_baseline_audit.py @@ -12,7 +12,6 @@ from __future__ import annotations import ast -import os import subprocess import sys import tempfile @@ -64,13 +63,12 @@ def run_audit(workflows, strict=False): (target / name).write_text(body, encoding="utf-8") summary = Path(tmp, "summary.md") summary.touch() - # Deliberately not os.environ: the script under test is read out of a - # workflow file that a pull request can modify, so it is given only the - # two variables it reads and nothing else to disclose. + # Deliberately not the inherited environment: the script under test is + # read out of a workflow file that a pull request can modify, so it gets + # only the two variables it reads and nothing else to disclose. env = { "STRICT": "true" if strict else "false", "GITHUB_STEP_SUMMARY": str(summary), - "PATH": os.environ.get("PATH", ""), } proc = subprocess.run( [sys.executable, "-c", audit_source()],