From 55e5a1eeaf665d530af1b38ffc683a606dfa2a99 Mon Sep 17 00:00:00 2001 From: divyansha Date: Sun, 24 May 2026 00:10:51 +0530 Subject: [PATCH 1/8] feat(ci): add changed-file scoped test selection with full-suite fallback --- .github/workflows/ci.yml | 58 ++++++++-- scripts/select_tests.py | 132 ++++++++++++++++++++++ testing/backend/unit/test_select_tests.py | 72 ++++++++++++ 3 files changed, 250 insertions(+), 12 deletions(-) create mode 100755 scripts/select_tests.py create mode 100644 testing/backend/unit/test_select_tests.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cd3e251..86874186 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,42 @@ permissions: contents: read jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + run_backend: ${{ steps.filter.outputs.run_backend }} + run_frontend: ${{ steps.filter.outputs.run_frontend }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Fetch base branch for diff + if: github.event_name == 'pull_request' + run: git fetch origin "${{ github.base_ref }}" --depth=1 + - name: Determine test selection + id: filter + run: python3 scripts/select_tests.py + + formatting-hygiene: + needs: detect-changes + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check formatting hygiene on changed files + run: | + git fetch origin "${{ github.base_ref }}" --depth=1 + git diff --check "origin/${{ github.base_ref }}"...HEAD + backend-lint: + needs: detect-changes + if: needs.detect-changes.outputs.run_backend == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -25,6 +60,12 @@ jobs: run: ruff check backend testing/backend backend-tests: + needs: [detect-changes, backend-lint, formatting-hygiene] + if: | + always() && + needs.detect-changes.outputs.run_backend == 'true' && + (needs.backend-lint.result == 'success' || needs.backend-lint.result == 'skipped') && + (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -41,6 +82,11 @@ jobs: run: pytest testing/backend -q frontend-checks: + needs: [detect-changes, formatting-hygiene] + if: | + always() && + needs.detect-changes.outputs.run_frontend == 'true' && + (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped') runs-on: ubuntu-latest defaults: run: @@ -66,15 +112,3 @@ jobs: run: npm run test - name: Build frontend run: npm run build - - formatting-hygiene: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Check formatting hygiene on changed files - run: | - git fetch origin "${{ github.base_ref }}" --depth=1 - git diff --check "origin/${{ github.base_ref }}"...HEAD diff --git a/scripts/select_tests.py b/scripts/select_tests.py new file mode 100755 index 00000000..44abb42c --- /dev/null +++ b/scripts/select_tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +import os +import sys +import argparse +import subprocess + +def get_changed_files(): + """ + Attempts to detect changed files using git diff. + """ + base_ref = os.environ.get("GITHUB_BASE_REF", "main") + # Commands to try in order of preference + commands = [ + ["git", "diff", "--name-only", f"origin/{base_ref}...HEAD"], + ["git", "diff", "--name-only", f"{base_ref}...HEAD"], + ["git", "diff", "--name-only", "HEAD~1"], + ["git", "diff", "--name-only"], + ] + for cmd in commands: + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + files = [line.strip() for line in res.stdout.splitlines() if line.strip()] + if files: + return files + except Exception: + continue + return [] + +def classify_file(filepath): + """ + Classifies a file path into a logical CI category. + """ + filepath = filepath.strip() + if not filepath: + return "DOCS" + + # Convert backslashes to forward slashes for cross-platform robustness + filepath = filepath.replace("\\", "/") + + # Check for docs first (both .md files anywhere and anything in docs/ directory) + if filepath.endswith(".md") or filepath.startswith("docs/"): + return "DOCS" + + # Check for frontend files + if filepath.startswith("frontend/"): + return "FRONTEND" + + # Check for plugin files + if filepath.startswith("plugins/"): + return "PLUGINS" + + # Check for backend files + if ( + filepath.startswith("backend/") + or filepath.startswith("testing/backend/") + or filepath == "pyproject.toml" + or (filepath.startswith("scripts/") and not filepath.endswith("check-artifacts.sh")) + ): + return "BACKEND" + + # Any other files (root scripts, github workflows, config files) + return "SHARED_OR_CONFIG" + +def select_tests(files): + """ + Decides which test suites to run based on a list of changed files. + Returns: (run_backend, run_frontend) + """ + if not files: + # Fall back to running the full suite to be safe + return True, True + + categories = {classify_file(f) for f in files} + + # If any changed file is SHARED_OR_CONFIG, run the full suite + if "SHARED_OR_CONFIG" in categories: + return True, True + + # If there are both BACKEND and FRONTEND changes, run the full suite + if "BACKEND" in categories and "FRONTEND" in categories: + return True, True + + run_backend = False + run_frontend = False + + # If BACKEND or PLUGINS changed, run backend tests + if "BACKEND" in categories or "PLUGINS" in categories: + run_backend = True + + # If FRONTEND changed, run frontend tests + if "FRONTEND" in categories: + run_frontend = True + + return run_backend, run_frontend + +def write_outputs(run_backend, run_frontend): + """ + Writes GITHUB_OUTPUT variables if the file exists, or prints to stdout. + """ + output_file = os.environ.get("GITHUB_OUTPUT") + backend_str = "true" if run_backend else "false" + frontend_str = "true" if run_frontend else "false" + + if output_file: + with open(output_file, "a") as f: + f.write(f"run_backend={backend_str}\n") + f.write(f"run_frontend={frontend_str}\n") + print(f"Written to GITHUB_OUTPUT: run_backend={backend_str}, run_frontend={frontend_str}") + else: + print(f"run_backend={backend_str}") + print(f"run_frontend={frontend_str}") + +def main(): + parser = argparse.ArgumentParser(description="Determine which tests to run based on changed files.") + parser.add_argument( + "--files", + nargs="*", + help="List of changed files. If not specified, git diff will be used to detect changes.", + ) + args = parser.parse_args() + + if args.files is not None: + files = args.files + else: + files = get_changed_files() + print(f"Detected changed files: {files}") + + run_backend, run_frontend = select_tests(files) + write_outputs(run_backend, run_frontend) + +if __name__ == "__main__": + main() diff --git a/testing/backend/unit/test_select_tests.py b/testing/backend/unit/test_select_tests.py new file mode 100644 index 00000000..f7dbe73d --- /dev/null +++ b/testing/backend/unit/test_select_tests.py @@ -0,0 +1,72 @@ +import os +import sys + +# Add root directory to sys.path so we can import from scripts +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) + +from scripts.select_tests import classify_file, select_tests + +def test_classify_file_backend(): + assert classify_file("backend/secuscan/main.py") == "BACKEND" + assert classify_file("testing/backend/unit/test_select_tests.py") == "BACKEND" + assert classify_file("pyproject.toml") == "BACKEND" + assert classify_file("scripts/refresh_plugin_checksum.py") == "BACKEND" + +def test_classify_file_frontend(): + assert classify_file("frontend/src/App.tsx") == "FRONTEND" + assert classify_file("frontend/package.json") == "FRONTEND" + +def test_classify_file_plugins(): + assert classify_file("plugins/nmap/metadata.json") == "PLUGINS" + assert classify_file("plugins/zap/parser.py") == "PLUGINS" + +def test_classify_file_docs(): + assert classify_file("README.md") == "DOCS" + assert classify_file("docs/architecture.md") == "DOCS" + assert classify_file("backend/README.md") == "DOCS" + +def test_classify_file_shared_or_config(): + assert classify_file(".github/workflows/ci.yml") == "SHARED_OR_CONFIG" + assert classify_file(".gitignore") == "SHARED_OR_CONFIG" + assert classify_file("setup.sh") == "SHARED_OR_CONFIG" + assert classify_file("docker-compose.yml") == "SHARED_OR_CONFIG" + assert classify_file("scripts/check-artifacts.sh") == "SHARED_OR_CONFIG" + +def test_select_tests_empty(): + # Empty changed file list must fall back to running everything + assert select_tests([]) == (True, True) + +def test_select_tests_backend_only(): + assert select_tests(["backend/secuscan/main.py"]) == (True, False) + assert select_tests(["backend/secuscan/main.py", "testing/backend/unit/test_models.py"]) == (True, False) + +def test_select_tests_frontend_only(): + assert select_tests(["frontend/src/App.tsx"]) == (False, True) + assert select_tests(["frontend/src/App.tsx", "frontend/package.json"]) == (False, True) + +def test_select_tests_plugins_only(): + # Plugins only should run backend tests + assert select_tests(["plugins/nmap/metadata.json"]) == (True, False) + +def test_select_tests_docs_only(): + # Docs only should run no code tests + assert select_tests(["README.md"]) == (False, False) + assert select_tests(["README.md", "docs/architecture.md"]) == (False, False) + +def test_select_tests_mixed_backend_frontend(): + # Mixed backend and frontend changes should run everything + assert select_tests(["backend/secuscan/main.py", "frontend/src/App.tsx"]) == (True, True) + +def test_select_tests_mixed_backend_plugins(): + # Backend + plugins should only run backend tests + assert select_tests(["backend/secuscan/main.py", "plugins/nmap/metadata.json"]) == (True, False) + +def test_select_tests_mixed_frontend_docs(): + # Frontend + docs should only run frontend tests + assert select_tests(["frontend/src/App.tsx", "README.md"]) == (False, True) + +def test_select_tests_shared_config_fallback(): + # Any shared config file triggers full suite fallback + assert select_tests([".github/workflows/ci.yml"]) == (True, True) + assert select_tests(["setup.sh", "backend/secuscan/main.py"]) == (True, True) + assert select_tests([".gitignore", "frontend/src/App.tsx"]) == (True, True) From 95715bcffde5df65ec090a339b3a4a34c944d3d8 Mon Sep 17 00:00:00 2001 From: divyansha Date: Sun, 24 May 2026 00:23:30 +0530 Subject: [PATCH 2/8] style(ci): format select_tests and fix trailing whitespaces --- scripts/select_tests.py | 57 ++++++++++++++--------- testing/backend/unit/test_select_tests.py | 37 +++++++++++++-- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 44abb42c..097dbf48 100755 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -4,6 +4,7 @@ import argparse import subprocess + def get_changed_files(): """ Attempts to detect changed files using git diff. @@ -26,6 +27,7 @@ def get_changed_files(): continue return [] + def classify_file(filepath): """ Classifies a file path into a logical CI category. @@ -33,34 +35,38 @@ def classify_file(filepath): filepath = filepath.strip() if not filepath: return "DOCS" - + # Convert backslashes to forward slashes for cross-platform robustness filepath = filepath.replace("\\", "/") - + # Check for docs first (both .md files anywhere and anything in docs/ directory) if filepath.endswith(".md") or filepath.startswith("docs/"): return "DOCS" - + # Check for frontend files if filepath.startswith("frontend/"): return "FRONTEND" - + # Check for plugin files if filepath.startswith("plugins/"): return "PLUGINS" - + # Check for backend files if ( - filepath.startswith("backend/") - or filepath.startswith("testing/backend/") - or filepath == "pyproject.toml" - or (filepath.startswith("scripts/") and not filepath.endswith("check-artifacts.sh")) + filepath.startswith("backend/") + or filepath.startswith("testing/backend/") + or filepath == "pyproject.toml" + or ( + filepath.startswith("scripts/") + and not filepath.endswith("check-artifacts.sh") + ) ): return "BACKEND" - + # Any other files (root scripts, github workflows, config files) return "SHARED_OR_CONFIG" + def select_tests(files): """ Decides which test suites to run based on a list of changed files. @@ -69,30 +75,31 @@ def select_tests(files): if not files: # Fall back to running the full suite to be safe return True, True - + categories = {classify_file(f) for f in files} - + # If any changed file is SHARED_OR_CONFIG, run the full suite if "SHARED_OR_CONFIG" in categories: return True, True - + # If there are both BACKEND and FRONTEND changes, run the full suite if "BACKEND" in categories and "FRONTEND" in categories: return True, True - + run_backend = False run_frontend = False - + # If BACKEND or PLUGINS changed, run backend tests if "BACKEND" in categories or "PLUGINS" in categories: run_backend = True - + # If FRONTEND changed, run frontend tests if "FRONTEND" in categories: run_frontend = True - + return run_backend, run_frontend + def write_outputs(run_backend, run_frontend): """ Writes GITHUB_OUTPUT variables if the file exists, or prints to stdout. @@ -100,33 +107,39 @@ def write_outputs(run_backend, run_frontend): output_file = os.environ.get("GITHUB_OUTPUT") backend_str = "true" if run_backend else "false" frontend_str = "true" if run_frontend else "false" - + if output_file: with open(output_file, "a") as f: f.write(f"run_backend={backend_str}\n") f.write(f"run_frontend={frontend_str}\n") - print(f"Written to GITHUB_OUTPUT: run_backend={backend_str}, run_frontend={frontend_str}") + print( + f"Written to GITHUB_OUTPUT: run_backend={backend_str}, run_frontend={frontend_str}" + ) else: print(f"run_backend={backend_str}") print(f"run_frontend={frontend_str}") + def main(): - parser = argparse.ArgumentParser(description="Determine which tests to run based on changed files.") + parser = argparse.ArgumentParser( + description="Determine which tests to run based on changed files." + ) parser.add_argument( "--files", nargs="*", help="List of changed files. If not specified, git diff will be used to detect changes.", ) args = parser.parse_args() - + if args.files is not None: files = args.files else: files = get_changed_files() print(f"Detected changed files: {files}") - + run_backend, run_frontend = select_tests(files) write_outputs(run_backend, run_frontend) + if __name__ == "__main__": main() diff --git a/testing/backend/unit/test_select_tests.py b/testing/backend/unit/test_select_tests.py index f7dbe73d..18da9a9b 100644 --- a/testing/backend/unit/test_select_tests.py +++ b/testing/backend/unit/test_select_tests.py @@ -2,29 +2,36 @@ import sys # Add root directory to sys.path so we can import from scripts -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")) +) from scripts.select_tests import classify_file, select_tests + def test_classify_file_backend(): assert classify_file("backend/secuscan/main.py") == "BACKEND" assert classify_file("testing/backend/unit/test_select_tests.py") == "BACKEND" assert classify_file("pyproject.toml") == "BACKEND" assert classify_file("scripts/refresh_plugin_checksum.py") == "BACKEND" + def test_classify_file_frontend(): assert classify_file("frontend/src/App.tsx") == "FRONTEND" assert classify_file("frontend/package.json") == "FRONTEND" + def test_classify_file_plugins(): assert classify_file("plugins/nmap/metadata.json") == "PLUGINS" assert classify_file("plugins/zap/parser.py") == "PLUGINS" + def test_classify_file_docs(): assert classify_file("README.md") == "DOCS" assert classify_file("docs/architecture.md") == "DOCS" assert classify_file("backend/README.md") == "DOCS" + def test_classify_file_shared_or_config(): assert classify_file(".github/workflows/ci.yml") == "SHARED_OR_CONFIG" assert classify_file(".gitignore") == "SHARED_OR_CONFIG" @@ -32,39 +39,59 @@ def test_classify_file_shared_or_config(): assert classify_file("docker-compose.yml") == "SHARED_OR_CONFIG" assert classify_file("scripts/check-artifacts.sh") == "SHARED_OR_CONFIG" + def test_select_tests_empty(): # Empty changed file list must fall back to running everything assert select_tests([]) == (True, True) + def test_select_tests_backend_only(): assert select_tests(["backend/secuscan/main.py"]) == (True, False) - assert select_tests(["backend/secuscan/main.py", "testing/backend/unit/test_models.py"]) == (True, False) + assert select_tests( + ["backend/secuscan/main.py", "testing/backend/unit/test_models.py"] + ) == (True, False) + def test_select_tests_frontend_only(): assert select_tests(["frontend/src/App.tsx"]) == (False, True) - assert select_tests(["frontend/src/App.tsx", "frontend/package.json"]) == (False, True) + assert select_tests(["frontend/src/App.tsx", "frontend/package.json"]) == ( + False, + True, + ) + def test_select_tests_plugins_only(): # Plugins only should run backend tests assert select_tests(["plugins/nmap/metadata.json"]) == (True, False) + def test_select_tests_docs_only(): # Docs only should run no code tests assert select_tests(["README.md"]) == (False, False) assert select_tests(["README.md", "docs/architecture.md"]) == (False, False) + def test_select_tests_mixed_backend_frontend(): # Mixed backend and frontend changes should run everything - assert select_tests(["backend/secuscan/main.py", "frontend/src/App.tsx"]) == (True, True) + assert select_tests(["backend/secuscan/main.py", "frontend/src/App.tsx"]) == ( + True, + True, + ) + def test_select_tests_mixed_backend_plugins(): # Backend + plugins should only run backend tests - assert select_tests(["backend/secuscan/main.py", "plugins/nmap/metadata.json"]) == (True, False) + assert select_tests(["backend/secuscan/main.py", "plugins/nmap/metadata.json"]) == ( + True, + False, + ) + def test_select_tests_mixed_frontend_docs(): # Frontend + docs should only run frontend tests assert select_tests(["frontend/src/App.tsx", "README.md"]) == (False, True) + def test_select_tests_shared_config_fallback(): # Any shared config file triggers full suite fallback assert select_tests([".github/workflows/ci.yml"]) == (True, True) From fd7b7dfaa49c7df53b5cf197184a9d576e7d977d Mon Sep 17 00:00:00 2001 From: divyansha Date: Tue, 26 May 2026 16:08:16 +0530 Subject: [PATCH 3/8] docs(ci): describe changed-file scoped CI selection and fallback rules --- docs/ci-test-selection.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/ci-test-selection.md diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md new file mode 100644 index 00000000..569e747d --- /dev/null +++ b/docs/ci-test-selection.md @@ -0,0 +1,29 @@ +## CI Test Selection (Changed-file scoped) + +Purpose +------- +This document explains the changed-file scoped CI test selection implemented in `scripts/select_tests.py`. + +Behavior +-------- +- The script maps changed files to test subsets (backend, frontend, plugin tests, etc.). +- When a change touches only files mapped to a subset, CI runs that subset to save time. +- A full-suite fallback is used in these cases: + - Repository-level shared config changes (eg: changes to `pyproject.toml`, `.github/`, or shared CI config files). + - Documentation-only or push types explicitly configured to force full CI. + - When the mapping cannot classify changed files deterministically. + +Why this is safe +--------------- +- The fallback exists to guarantee full verification when a change may affect CI orchestration or multiple subsystems. +- The mapping is conservative: it prefers running more tests rather than skipping them. + +How to run locally +------------------- +Run the selection tool and dry-run tests locally: + +```bash +./venv_tests/bin/python scripts/select_tests.py --dry-run --changed-files +``` + +If you need this policy changed or to add new mappings, update `scripts/select_tests.py` and the unit tests under `testing/backend/unit/test_select_tests.py`. From 6ca55e62d5566ba35002904499ca5b25b71fb47c Mon Sep 17 00:00:00 2001 From: divyansha Date: Sat, 30 May 2026 12:16:00 +0530 Subject: [PATCH 4/8] fix(ci): handle PR vs push events for required checks safety --- .github/workflows/ci.yml | 2 + docs/ci-test-selection.md | 120 ++++++++++++++++++---- scripts/select_tests.py | 66 ++++++++++-- testing/backend/unit/test_select_tests.py | 33 ++++++ 4 files changed, 194 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da34d077..1a97f397 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: run: git fetch origin "${{ github.base_ref }}" --depth=1 - name: Determine test selection id: filter + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} run: python3 scripts/select_tests.py formatting-hygiene: diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index 569e747d..290949ca 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -4,26 +4,108 @@ Purpose ------- This document explains the changed-file scoped CI test selection implemented in `scripts/select_tests.py`. -Behavior --------- -- The script maps changed files to test subsets (backend, frontend, plugin tests, etc.). -- When a change touches only files mapped to a subset, CI runs that subset to save time. -- A full-suite fallback is used in these cases: - - Repository-level shared config changes (eg: changes to `pyproject.toml`, `.github/`, or shared CI config files). - - Documentation-only or push types explicitly configured to force full CI. - - When the mapping cannot classify changed files deterministically. - -Why this is safe ---------------- -- The fallback exists to guarantee full verification when a change may affect CI orchestration or multiple subsystems. -- The mapping is conservative: it prefers running more tests rather than skipping them. - -How to run locally -------------------- -Run the selection tool and dry-run tests locally: +## Branch Protection Safety Guarantee + +**CRITICAL:** This implementation is safe for use with GitHub branch protection because: + +1. **Pull Requests Always Run Full Suite** + - All PR changes ALWAYS trigger the full test suite (backend + frontend) + - This ensures required checks configured in branch protection never get marked as "skipped" + - A skipped required check would incorrectly block the PR merge + - Example: A docs-only PR still runs all tests to satisfy required checks + +2. **Push Events Use Selective Skipping** + - Only push events (commits to main/develop) use selective skipping + - Push events are informational and don't block merges + - Selective skipping saves CI time on post-merge verification + - Example: A docs-only commit to main skips tests safely (no blocking rules) + +## Event Type Behavior + +| Event | Docs-Only | Backend Only | Frontend Only | Mixed / Config | +|--------------|-----------|--------------|---------------|-----------------| +| `pull_request` | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite | +| `push` | ✗ Skip All | ✓ Backend | ✓ Frontend | ✓ Full Suite | + +Legend: ✓ = runs tests, ✗ = skips tests + +## File Classification + +The script classifies changed files into these categories: + +- **DOCS** (skippable): `.md` files anywhere, `docs/` directory + - Safe to skip because docs cannot affect code behavior + - Exception: In PRs, still runs full suite for branch protection + +- **FRONTEND**: `frontend/` directory + - Runs frontend tests only (unless mixed with backend) + +- **BACKEND**: `backend/`, `testing/backend/`, `pyproject.toml`, `scripts/*.py` + - Runs backend tests; includes plugins via plugin dependencies + +- **PLUGINS**: `plugins/` directory + - Treated as backend changes (runs backend tests) + +- **SHARED_OR_CONFIG** (forces full suite): `.github/`, root scripts, config files + - `.github/workflows/` changes → full suite (CI behavior changes) + - `setup.sh`, `docker-compose.yml` → full suite (system changes) + - These affect multiple subsystems and must be fully tested + +## Why This is Safe + +1. **Conservative Fallbacks** + - When in doubt, we run the full suite + - Docs-only + shared config → full suite + - Backend + frontend → full suite + - Mixed categories → full suite + +2. **Branch Protection Guaranteed** + - PRs cannot skip required checks (always full suite) + - Developers cannot accidentally merge untested code + - Required checks will pass/fail, never be skipped + +3. **Deterministic Classification** + - File paths are mapped consistently + - No heuristics or guessing + - Can be verified locally + +## Configuration + +To modify the test selection policy: + +1. Update file classification in `scripts/select_tests.py` → `classify_file()` +2. Update logic in `select_tests()` function +3. Add corresponding unit tests in `testing/backend/unit/test_select_tests.py` +4. Run tests locally: `pytest testing/backend/unit/test_select_tests.py -v` +5. Update this document if behavior changes + +## Local Testing + +Dry-run the selection tool locally: ```bash -./venv_tests/bin/python scripts/select_tests.py --dry-run --changed-files +# Test with specific files (simulating a PR/push) +python3 scripts/select_tests.py --files backend/main.py frontend/App.tsx --event-name pull_request +# Output: run_backend=true, run_frontend=true (PR always runs full suite) + +python3 scripts/select_tests.py --files README.md --event-name push +# Output: run_backend=false, run_frontend=false (docs-only push skips tests) + +python3 scripts/select_tests.py --files .github/workflows/ci.yml +# Output: run_backend=true, run_frontend=true (config changes run full suite) ``` -If you need this policy changed or to add new mappings, update `scripts/select_tests.py` and the unit tests under `testing/backend/unit/test_select_tests.py`. +## Required Checks in GitHub + +For this to work correctly, configure your branch protection rule on `main` to require these checks: + +- `formatting-hygiene` (always runs, PR-only) +- `backend-lint` (skippable based on changes) +- `backend-tests` (skippable based on changes) +- `frontend-checks` (skippable based on changes) + +This allows: +- ✅ Docs-only PR → required checks (formatting) run, optional checks (tests) run full suite +- ✅ Backend-only PR → full suite runs, satisfying all required checks +- ✅ Docs-only push → tests skip safely (push checks are not required) + diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 097dbf48..98830689 100755 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -28,9 +28,24 @@ def get_changed_files(): return [] +def get_event_name(): + """ + Get the GitHub event name (push or pull_request). + Defaults to 'push' for local/unknown environments. + """ + return os.environ.get("GITHUB_EVENT_NAME", "push") + + def classify_file(filepath): """ Classifies a file path into a logical CI category. + + Categories: + - DOCS: Documentation files (.md) - safe to skip for selective testing + - FRONTEND: Frontend code and tests + - PLUGINS: Plugin definitions + - BACKEND: Backend code, tests, and Python configs + - SHARED_OR_CONFIG: Shared configuration and CI workflow files - always requires full suite """ filepath = filepath.strip() if not filepath: @@ -40,6 +55,7 @@ def classify_file(filepath): filepath = filepath.replace("\\", "/") # Check for docs first (both .md files anywhere and anything in docs/ directory) + # SAFE to skip: documentation cannot affect code behavior if filepath.endswith(".md") or filepath.startswith("docs/"): return "DOCS" @@ -64,25 +80,51 @@ def classify_file(filepath): return "BACKEND" # Any other files (root scripts, github workflows, config files) + # UNSAFE to skip: these files affect CI behavior and shared configuration return "SHARED_OR_CONFIG" -def select_tests(files): +def select_tests(files, event_name="push"): """ - Decides which test suites to run based on a list of changed files. - Returns: (run_backend, run_frontend) + Decides which test suites to run based on changed files and event type. + + Args: + files: List of changed file paths + event_name: GitHub event type ('pull_request' or 'push') + + Returns: + Tuple of (run_backend: bool, run_frontend: bool) + + Logic: + ------ + For PULL REQUESTS (PR checks are required for merge): + - Always run full suite to ensure required checks pass + - This prevents required checks from being marked "skipped" in branch protection + - PR must be thoroughly tested before merge + + For PUSH events (push checks are informational): + - Use selective skipping to save CI time on main/develop + - Skip tests for docs-only changes + - Still run full suite for shared config changes """ if not files: - # Fall back to running the full suite to be safe + # Empty file list: fall back to running full suite to be safe return True, True + # CRITICAL: For pull requests, always run full suite + # This ensures branch protection required checks pass (not skipped) + if event_name == "pull_request": + return True, True + + # For push events, use selective skipping to optimize CI time categories = {classify_file(f) for f in files} - # If any changed file is SHARED_OR_CONFIG, run the full suite + # If any changed file is SHARED_OR_CONFIG, run full suite + # These files affect CI behavior and must be thoroughly tested if "SHARED_OR_CONFIG" in categories: return True, True - # If there are both BACKEND and FRONTEND changes, run the full suite + # If there are both BACKEND and FRONTEND changes, run full suite if "BACKEND" in categories and "FRONTEND" in categories: return True, True @@ -122,13 +164,18 @@ def write_outputs(run_backend, run_frontend): def main(): parser = argparse.ArgumentParser( - description="Determine which tests to run based on changed files." + description="Determine which tests to run based on changed files and event type." ) parser.add_argument( "--files", nargs="*", help="List of changed files. If not specified, git diff will be used to detect changes.", ) + parser.add_argument( + "--event-name", + default=None, + help="GitHub event name (pull_request or push). If not specified, reads from GITHUB_EVENT_NAME env var.", + ) args = parser.parse_args() if args.files is not None: @@ -137,7 +184,10 @@ def main(): files = get_changed_files() print(f"Detected changed files: {files}") - run_backend, run_frontend = select_tests(files) + event_name = args.event_name or get_event_name() + print(f"Event type: {event_name}") + + run_backend, run_frontend = select_tests(files, event_name=event_name) write_outputs(run_backend, run_frontend) diff --git a/testing/backend/unit/test_select_tests.py b/testing/backend/unit/test_select_tests.py index 18da9a9b..e8d4db8b 100644 --- a/testing/backend/unit/test_select_tests.py +++ b/testing/backend/unit/test_select_tests.py @@ -97,3 +97,36 @@ def test_select_tests_shared_config_fallback(): assert select_tests([".github/workflows/ci.yml"]) == (True, True) assert select_tests(["setup.sh", "backend/secuscan/main.py"]) == (True, True) assert select_tests([".gitignore", "frontend/src/App.tsx"]) == (True, True) + + +# ── Event-based logic (PR vs push) ──────────────────────────────────────────── + +def test_select_tests_pull_request_always_full_suite(): + """ + PRs must always run full suite to ensure required branch protection checks + do not get marked as 'skipped'. + """ + # Docs-only PR: still run full suite for safety + assert select_tests(["README.md"], event_name="pull_request") == (True, True) + # Backend-only PR: full suite (always) + assert select_tests(["backend/secuscan/main.py"], event_name="pull_request") == ( + True, + True, + ) + # Frontend-only PR: full suite (always) + assert select_tests(["frontend/src/App.tsx"], event_name="pull_request") == ( + True, + True, + ) + + +def test_select_tests_push_uses_selective_skipping(): + """ + Push events (to main) can use selective skipping to optimize CI time. + """ + # Push with docs-only: skip tests + assert select_tests(["README.md"], event_name="push") == (False, False) + # Push with backend: run backend only + assert select_tests(["backend/secuscan/main.py"], event_name="push") == (True, False) + # Push with frontend: run frontend only + assert select_tests(["frontend/src/App.tsx"], event_name="push") == (False, True) From fab7421e83a3de2ce86680d5c1cfc77734f4feff Mon Sep 17 00:00:00 2001 From: divyansha Date: Sat, 30 May 2026 12:25:27 +0530 Subject: [PATCH 5/8] style: remove trailing whitespace --- docs/ci-test-selection.md | 11 +++++------ scripts/select_tests.py | 10 +++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md index 290949ca..01042c75 100644 --- a/docs/ci-test-selection.md +++ b/docs/ci-test-selection.md @@ -8,13 +8,13 @@ This document explains the changed-file scoped CI test selection implemented in **CRITICAL:** This implementation is safe for use with GitHub branch protection because: -1. **Pull Requests Always Run Full Suite** +1. **Pull Requests Always Run Full Suite** - All PR changes ALWAYS trigger the full test suite (backend + frontend) - This ensures required checks configured in branch protection never get marked as "skipped" - A skipped required check would incorrectly block the PR merge - Example: A docs-only PR still runs all tests to satisfy required checks -2. **Push Events Use Selective Skipping** +2. **Push Events Use Selective Skipping** - Only push events (commits to main/develop) use selective skipping - Push events are informational and don't block merges - Selective skipping saves CI time on post-merge verification @@ -53,18 +53,18 @@ The script classifies changed files into these categories: ## Why This is Safe -1. **Conservative Fallbacks** +1. **Conservative Fallbacks** - When in doubt, we run the full suite - Docs-only + shared config → full suite - Backend + frontend → full suite - Mixed categories → full suite -2. **Branch Protection Guaranteed** +2. **Branch Protection Guaranteed** - PRs cannot skip required checks (always full suite) - Developers cannot accidentally merge untested code - Required checks will pass/fail, never be skipped -3. **Deterministic Classification** +3. **Deterministic Classification** - File paths are mapped consistently - No heuristics or guessing - Can be verified locally @@ -108,4 +108,3 @@ This allows: - ✅ Docs-only PR → required checks (formatting) run, optional checks (tests) run full suite - ✅ Backend-only PR → full suite runs, satisfying all required checks - ✅ Docs-only push → tests skip safely (push checks are not required) - diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 98830689..aeaf2005 100755 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -39,7 +39,7 @@ def get_event_name(): def classify_file(filepath): """ Classifies a file path into a logical CI category. - + Categories: - DOCS: Documentation files (.md) - safe to skip for selective testing - FRONTEND: Frontend code and tests @@ -87,21 +87,21 @@ def classify_file(filepath): def select_tests(files, event_name="push"): """ Decides which test suites to run based on changed files and event type. - + Args: files: List of changed file paths event_name: GitHub event type ('pull_request' or 'push') - + Returns: Tuple of (run_backend: bool, run_frontend: bool) - + Logic: ------ For PULL REQUESTS (PR checks are required for merge): - Always run full suite to ensure required checks pass - This prevents required checks from being marked "skipped" in branch protection - PR must be thoroughly tested before merge - + For PUSH events (push checks are informational): - Use selective skipping to save CI time on main/develop - Skip tests for docs-only changes From d67c6e74da443c296c60cc4baacb403a90e1feb5 Mon Sep 17 00:00:00 2001 From: divyansha Date: Mon, 1 Jun 2026 10:49:46 +0530 Subject: [PATCH 6/8] docs(ci): add proof of test selection behavior --- ci_proof.txt | 121 +++++++++++++++++++++++++++++++++++++++ scripts/run_ci_proofs.sh | 51 +++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 ci_proof.txt create mode 100755 scripts/run_ci_proofs.sh diff --git a/ci_proof.txt b/ci_proof.txt new file mode 100644 index 00000000..7e495582 --- /dev/null +++ b/ci_proof.txt @@ -0,0 +1,121 @@ +## CI Test Selection Proof +Generated: Mon Jun 1 05:16:47 UTC 2026 +This file proves that the test selection logic correctly handles all critical scenarios. + +---------------------------------------------------------------------- +SCENARIO: PR with docs-only change +EVENT: pull_request +FILES: README.md + +COMMAND: +python3 scripts/select_tests.py --files README.md --event-name pull_request + +OUTPUT: +Event type: pull_request +run_backend=true +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: PR with backend-only change +EVENT: pull_request +FILES: backend/secuscan/main.py + +COMMAND: +python3 scripts/select_tests.py --files backend/secuscan/main.py --event-name pull_request + +OUTPUT: +Event type: pull_request +run_backend=true +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: PR with frontend-only change +EVENT: pull_request +FILES: frontend/src/App.tsx + +COMMAND: +python3 scripts/select_tests.py --files frontend/src/App.tsx --event-name pull_request + +OUTPUT: +Event type: pull_request +run_backend=true +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: PR with shared config change +EVENT: pull_request +FILES: .github/workflows/ci.yml + +COMMAND: +python3 scripts/select_tests.py --files .github/workflows/ci.yml --event-name pull_request + +OUTPUT: +Event type: pull_request +run_backend=true +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: Push with docs-only change +EVENT: push +FILES: README.md + +COMMAND: +python3 scripts/select_tests.py --files README.md --event-name push + +OUTPUT: +Event type: push +run_backend=false +run_frontend=false + +---------------------------------------------------------------------- +SCENARIO: Push with backend-only change +EVENT: push +FILES: backend/secuscan/main.py + +COMMAND: +python3 scripts/select_tests.py --files backend/secuscan/main.py --event-name push + +OUTPUT: +Event type: push +run_backend=true +run_frontend=false + +---------------------------------------------------------------------- +SCENARIO: Push with frontend-only change +EVENT: push +FILES: frontend/src/App.tsx + +COMMAND: +python3 scripts/select_tests.py --files frontend/src/App.tsx --event-name push + +OUTPUT: +Event type: push +run_backend=false +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: Push with shared config change +EVENT: push +FILES: .github/workflows/ci.yml + +COMMAND: +python3 scripts/select_tests.py --files .github/workflows/ci.yml --event-name push + +OUTPUT: +Event type: push +run_backend=true +run_frontend=true + +---------------------------------------------------------------------- +SCENARIO: Push with backend and frontend change +EVENT: push +FILES: backend/secuscan/main.py frontend/src/App.tsx + +COMMAND: +python3 scripts/select_tests.py --files backend/secuscan/main.py frontend/src/App.tsx --event-name push + +OUTPUT: +Event type: push +run_backend=true +run_frontend=true + diff --git a/scripts/run_ci_proofs.sh b/scripts/run_ci_proofs.sh new file mode 100755 index 00000000..a9c4f999 --- /dev/null +++ b/scripts/run_ci_proofs.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# scripts/run_ci_proofs.sh + +set -euo pipefail + +# This script generates a proof file demonstrating that the test selection logic +# behaves as expected under different conditions. This is used to satisfy +# maintainer concerns about required checks being skipped. + +OUTPUT_FILE="ci_proof.txt" +SELECTOR_SCRIPT="scripts/select_tests.py" + +# Header +echo "## CI Test Selection Proof" > "$OUTPUT_FILE" +echo "Generated: $(date -u)" >> "$OUTPUT_FILE" +echo "This file proves that the test selection logic correctly handles all critical scenarios." >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +run_proof() { + local description="$1" + local files="$2" + local event_name="$3" + + echo "----------------------------------------------------------------------" >> "$OUTPUT_FILE" + echo "SCENARIO: $description" >> "$OUTPUT_FILE" + echo "EVENT: $event_name" >> "$OUTPUT_FILE" + echo "FILES: $files" >> "$OUTPUT_FILE" + echo "" >> "$OUTPUT_FILE" + echo "COMMAND:" >> "$OUTPUT_FILE" + echo "python3 $SELECTOR_SCRIPT --files $files --event-name $event_name" >> "$OUTPUT_FILE" + echo "" >> "$OUTPUT_FILE" + echo "OUTPUT:" >> "$OUTPUT_FILE" + # Run the command and append its output to the proof file + python3 "$SELECTOR_SCRIPT" --files $files --event-name "$event_name" >> "$OUTPUT_FILE" + echo "" >> "$OUTPUT_FILE" +} + +# --- PULL REQUEST SCENARIOS (ALWAYS RUN FULL SUITE) --- +run_proof "PR with docs-only change" "README.md" "pull_request" +run_proof "PR with backend-only change" "backend/secuscan/main.py" "pull_request" +run_proof "PR with frontend-only change" "frontend/src/App.tsx" "pull_request" +run_proof "PR with shared config change" ".github/workflows/ci.yml" "pull_request" + +# --- PUSH SCENARIOS (SELECTIVE SKIPPING) --- +run_proof "Push with docs-only change" "README.md" "push" +run_proof "Push with backend-only change" "backend/secuscan/main.py" "push" +run_proof "Push with frontend-only change" "frontend/src/App.tsx" "push" +run_proof "Push with shared config change" ".github/workflows/ci.yml" "push" +run_proof "Push with backend and frontend change" "backend/secuscan/main.py frontend/src/App.tsx" "push" + +echo "✓ CI proof generated at $OUTPUT_FILE" From cc088a8601c98a33db04f74836fbe1b4a2507585 Mon Sep 17 00:00:00 2001 From: divyansha Date: Mon, 1 Jun 2026 10:57:01 +0530 Subject: [PATCH 7/8] style(ci): fix trailing whitespace in proof script --- ci_proof.txt | 1 - scripts/run_ci_proofs.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ci_proof.txt b/ci_proof.txt index 7e495582..5c4f1ed2 100644 --- a/ci_proof.txt +++ b/ci_proof.txt @@ -118,4 +118,3 @@ OUTPUT: Event type: push run_backend=true run_frontend=true - diff --git a/scripts/run_ci_proofs.sh b/scripts/run_ci_proofs.sh index a9c4f999..faef8359 100755 --- a/scripts/run_ci_proofs.sh +++ b/scripts/run_ci_proofs.sh @@ -20,7 +20,7 @@ run_proof() { local description="$1" local files="$2" local event_name="$3" - + echo "----------------------------------------------------------------------" >> "$OUTPUT_FILE" echo "SCENARIO: $description" >> "$OUTPUT_FILE" echo "EVENT: $event_name" >> "$OUTPUT_FILE" From 416fb660d8158c3558c3886ced72c11a6f160055 Mon Sep 17 00:00:00 2001 From: Utkarsh Singh <183999732+utksh1@users.noreply.github.com> Date: Tue, 2 Jun 2026 00:27:53 +0530 Subject: [PATCH 8/8] chore(ci): remove generated proof artifacts --- ci_proof.txt | 120 --------------------------------------- scripts/run_ci_proofs.sh | 51 ----------------- 2 files changed, 171 deletions(-) delete mode 100644 ci_proof.txt delete mode 100755 scripts/run_ci_proofs.sh diff --git a/ci_proof.txt b/ci_proof.txt deleted file mode 100644 index 5c4f1ed2..00000000 --- a/ci_proof.txt +++ /dev/null @@ -1,120 +0,0 @@ -## CI Test Selection Proof -Generated: Mon Jun 1 05:16:47 UTC 2026 -This file proves that the test selection logic correctly handles all critical scenarios. - ----------------------------------------------------------------------- -SCENARIO: PR with docs-only change -EVENT: pull_request -FILES: README.md - -COMMAND: -python3 scripts/select_tests.py --files README.md --event-name pull_request - -OUTPUT: -Event type: pull_request -run_backend=true -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: PR with backend-only change -EVENT: pull_request -FILES: backend/secuscan/main.py - -COMMAND: -python3 scripts/select_tests.py --files backend/secuscan/main.py --event-name pull_request - -OUTPUT: -Event type: pull_request -run_backend=true -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: PR with frontend-only change -EVENT: pull_request -FILES: frontend/src/App.tsx - -COMMAND: -python3 scripts/select_tests.py --files frontend/src/App.tsx --event-name pull_request - -OUTPUT: -Event type: pull_request -run_backend=true -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: PR with shared config change -EVENT: pull_request -FILES: .github/workflows/ci.yml - -COMMAND: -python3 scripts/select_tests.py --files .github/workflows/ci.yml --event-name pull_request - -OUTPUT: -Event type: pull_request -run_backend=true -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: Push with docs-only change -EVENT: push -FILES: README.md - -COMMAND: -python3 scripts/select_tests.py --files README.md --event-name push - -OUTPUT: -Event type: push -run_backend=false -run_frontend=false - ----------------------------------------------------------------------- -SCENARIO: Push with backend-only change -EVENT: push -FILES: backend/secuscan/main.py - -COMMAND: -python3 scripts/select_tests.py --files backend/secuscan/main.py --event-name push - -OUTPUT: -Event type: push -run_backend=true -run_frontend=false - ----------------------------------------------------------------------- -SCENARIO: Push with frontend-only change -EVENT: push -FILES: frontend/src/App.tsx - -COMMAND: -python3 scripts/select_tests.py --files frontend/src/App.tsx --event-name push - -OUTPUT: -Event type: push -run_backend=false -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: Push with shared config change -EVENT: push -FILES: .github/workflows/ci.yml - -COMMAND: -python3 scripts/select_tests.py --files .github/workflows/ci.yml --event-name push - -OUTPUT: -Event type: push -run_backend=true -run_frontend=true - ----------------------------------------------------------------------- -SCENARIO: Push with backend and frontend change -EVENT: push -FILES: backend/secuscan/main.py frontend/src/App.tsx - -COMMAND: -python3 scripts/select_tests.py --files backend/secuscan/main.py frontend/src/App.tsx --event-name push - -OUTPUT: -Event type: push -run_backend=true -run_frontend=true diff --git a/scripts/run_ci_proofs.sh b/scripts/run_ci_proofs.sh deleted file mode 100755 index faef8359..00000000 --- a/scripts/run_ci_proofs.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# scripts/run_ci_proofs.sh - -set -euo pipefail - -# This script generates a proof file demonstrating that the test selection logic -# behaves as expected under different conditions. This is used to satisfy -# maintainer concerns about required checks being skipped. - -OUTPUT_FILE="ci_proof.txt" -SELECTOR_SCRIPT="scripts/select_tests.py" - -# Header -echo "## CI Test Selection Proof" > "$OUTPUT_FILE" -echo "Generated: $(date -u)" >> "$OUTPUT_FILE" -echo "This file proves that the test selection logic correctly handles all critical scenarios." >> "$OUTPUT_FILE" -echo "" >> "$OUTPUT_FILE" - -run_proof() { - local description="$1" - local files="$2" - local event_name="$3" - - echo "----------------------------------------------------------------------" >> "$OUTPUT_FILE" - echo "SCENARIO: $description" >> "$OUTPUT_FILE" - echo "EVENT: $event_name" >> "$OUTPUT_FILE" - echo "FILES: $files" >> "$OUTPUT_FILE" - echo "" >> "$OUTPUT_FILE" - echo "COMMAND:" >> "$OUTPUT_FILE" - echo "python3 $SELECTOR_SCRIPT --files $files --event-name $event_name" >> "$OUTPUT_FILE" - echo "" >> "$OUTPUT_FILE" - echo "OUTPUT:" >> "$OUTPUT_FILE" - # Run the command and append its output to the proof file - python3 "$SELECTOR_SCRIPT" --files $files --event-name "$event_name" >> "$OUTPUT_FILE" - echo "" >> "$OUTPUT_FILE" -} - -# --- PULL REQUEST SCENARIOS (ALWAYS RUN FULL SUITE) --- -run_proof "PR with docs-only change" "README.md" "pull_request" -run_proof "PR with backend-only change" "backend/secuscan/main.py" "pull_request" -run_proof "PR with frontend-only change" "frontend/src/App.tsx" "pull_request" -run_proof "PR with shared config change" ".github/workflows/ci.yml" "pull_request" - -# --- PUSH SCENARIOS (SELECTIVE SKIPPING) --- -run_proof "Push with docs-only change" "README.md" "push" -run_proof "Push with backend-only change" "backend/secuscan/main.py" "push" -run_proof "Push with frontend-only change" "frontend/src/App.tsx" "push" -run_proof "Push with shared config change" ".github/workflows/ci.yml" "push" -run_proof "Push with backend and frontend change" "backend/secuscan/main.py frontend/src/App.tsx" "push" - -echo "✓ CI proof generated at $OUTPUT_FILE"