diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46e0aa1a..1a97f397 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,44 @@ permissions: contents: read jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + run_backend: ${{ steps.filter.outputs.run_backend }} + run_frontend: ${{ steps.filter.outputs.run_frontend }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Fetch base branch for diff + if: github.event_name == 'pull_request' + run: git fetch origin "${{ github.base_ref }}" --depth=1 + - name: Determine test selection + id: filter + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + run: python3 scripts/select_tests.py + + formatting-hygiene: + needs: detect-changes + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check formatting hygiene on changed files + run: | + git fetch origin "${{ github.base_ref }}" --depth=1 + git diff --check "origin/${{ github.base_ref }}"...HEAD + backend-lint: + needs: detect-changes + if: needs.detect-changes.outputs.run_backend == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -25,6 +62,12 @@ jobs: run: ruff check backend testing/backend backend-tests: + needs: [detect-changes, backend-lint, formatting-hygiene] + if: | + always() && + needs.detect-changes.outputs.run_backend == 'true' && + (needs.backend-lint.result == 'success' || needs.backend-lint.result == 'skipped') && + (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -69,6 +112,11 @@ jobs: echo "::warning::Performance benchmark thresholds exceeded or benchmarks failed to run. Check the job logs for details." frontend-checks: + needs: [detect-changes, formatting-hygiene] + if: | + always() && + needs.detect-changes.outputs.run_frontend == 'true' && + (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped') runs-on: ubuntu-latest defaults: run: @@ -94,15 +142,3 @@ jobs: run: npm run test - name: Build frontend run: npm run build - - formatting-hygiene: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Check formatting hygiene on changed files - run: | - git fetch origin "${{ github.base_ref }}" --depth=1 - git diff --check "origin/${{ github.base_ref }}"...HEAD diff --git a/docs/ci-test-selection.md b/docs/ci-test-selection.md new file mode 100644 index 00000000..01042c75 --- /dev/null +++ b/docs/ci-test-selection.md @@ -0,0 +1,110 @@ +## CI Test Selection (Changed-file scoped) + +Purpose +------- +This document explains the changed-file scoped CI test selection implemented in `scripts/select_tests.py`. + +## Branch Protection Safety Guarantee + +**CRITICAL:** This implementation is safe for use with GitHub branch protection because: + +1. **Pull Requests Always Run Full Suite** + - All PR changes ALWAYS trigger the full test suite (backend + frontend) + - This ensures required checks configured in branch protection never get marked as "skipped" + - A skipped required check would incorrectly block the PR merge + - Example: A docs-only PR still runs all tests to satisfy required checks + +2. **Push Events Use Selective Skipping** + - Only push events (commits to main/develop) use selective skipping + - Push events are informational and don't block merges + - Selective skipping saves CI time on post-merge verification + - Example: A docs-only commit to main skips tests safely (no blocking rules) + +## Event Type Behavior + +| Event | Docs-Only | Backend Only | Frontend Only | Mixed / Config | +|--------------|-----------|--------------|---------------|-----------------| +| `pull_request` | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite | +| `push` | ✗ Skip All | ✓ Backend | ✓ Frontend | ✓ Full Suite | + +Legend: ✓ = runs tests, ✗ = skips tests + +## File Classification + +The script classifies changed files into these categories: + +- **DOCS** (skippable): `.md` files anywhere, `docs/` directory + - Safe to skip because docs cannot affect code behavior + - Exception: In PRs, still runs full suite for branch protection + +- **FRONTEND**: `frontend/` directory + - Runs frontend tests only (unless mixed with backend) + +- **BACKEND**: `backend/`, `testing/backend/`, `pyproject.toml`, `scripts/*.py` + - Runs backend tests; includes plugins via plugin dependencies + +- **PLUGINS**: `plugins/` directory + - Treated as backend changes (runs backend tests) + +- **SHARED_OR_CONFIG** (forces full suite): `.github/`, root scripts, config files + - `.github/workflows/` changes → full suite (CI behavior changes) + - `setup.sh`, `docker-compose.yml` → full suite (system changes) + - These affect multiple subsystems and must be fully tested + +## Why This is Safe + +1. **Conservative Fallbacks** + - When in doubt, we run the full suite + - Docs-only + shared config → full suite + - Backend + frontend → full suite + - Mixed categories → full suite + +2. **Branch Protection Guaranteed** + - PRs cannot skip required checks (always full suite) + - Developers cannot accidentally merge untested code + - Required checks will pass/fail, never be skipped + +3. **Deterministic Classification** + - File paths are mapped consistently + - No heuristics or guessing + - Can be verified locally + +## Configuration + +To modify the test selection policy: + +1. Update file classification in `scripts/select_tests.py` → `classify_file()` +2. Update logic in `select_tests()` function +3. Add corresponding unit tests in `testing/backend/unit/test_select_tests.py` +4. Run tests locally: `pytest testing/backend/unit/test_select_tests.py -v` +5. Update this document if behavior changes + +## Local Testing + +Dry-run the selection tool locally: + +```bash +# Test with specific files (simulating a PR/push) +python3 scripts/select_tests.py --files backend/main.py frontend/App.tsx --event-name pull_request +# Output: run_backend=true, run_frontend=true (PR always runs full suite) + +python3 scripts/select_tests.py --files README.md --event-name push +# Output: run_backend=false, run_frontend=false (docs-only push skips tests) + +python3 scripts/select_tests.py --files .github/workflows/ci.yml +# Output: run_backend=true, run_frontend=true (config changes run full suite) +``` + +## Required Checks in GitHub + +For this to work correctly, configure your branch protection rule on `main` to require these checks: + +- `formatting-hygiene` (always runs, PR-only) +- `backend-lint` (skippable based on changes) +- `backend-tests` (skippable based on changes) +- `frontend-checks` (skippable based on changes) + +This allows: +- ✅ Docs-only PR → required checks (formatting) run, optional checks (tests) run full suite +- ✅ Backend-only PR → full suite runs, satisfying all required checks +- ✅ Docs-only push → tests skip safely (push checks are not required) diff --git a/scripts/select_tests.py b/scripts/select_tests.py new file mode 100755 index 00000000..aeaf2005 --- /dev/null +++ b/scripts/select_tests.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +import os +import sys +import argparse +import subprocess + + +def get_changed_files(): + """ + Attempts to detect changed files using git diff. + """ + base_ref = os.environ.get("GITHUB_BASE_REF", "main") + # Commands to try in order of preference + commands = [ + ["git", "diff", "--name-only", f"origin/{base_ref}...HEAD"], + ["git", "diff", "--name-only", f"{base_ref}...HEAD"], + ["git", "diff", "--name-only", "HEAD~1"], + ["git", "diff", "--name-only"], + ] + for cmd in commands: + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + files = [line.strip() for line in res.stdout.splitlines() if line.strip()] + if files: + return files + except Exception: + continue + return [] + + +def get_event_name(): + """ + Get the GitHub event name (push or pull_request). + Defaults to 'push' for local/unknown environments. + """ + return os.environ.get("GITHUB_EVENT_NAME", "push") + + +def classify_file(filepath): + """ + Classifies a file path into a logical CI category. + + Categories: + - DOCS: Documentation files (.md) - safe to skip for selective testing + - FRONTEND: Frontend code and tests + - PLUGINS: Plugin definitions + - BACKEND: Backend code, tests, and Python configs + - SHARED_OR_CONFIG: Shared configuration and CI workflow files - always requires full suite + """ + filepath = filepath.strip() + if not filepath: + return "DOCS" + + # Convert backslashes to forward slashes for cross-platform robustness + filepath = filepath.replace("\\", "/") + + # Check for docs first (both .md files anywhere and anything in docs/ directory) + # SAFE to skip: documentation cannot affect code behavior + if filepath.endswith(".md") or filepath.startswith("docs/"): + return "DOCS" + + # Check for frontend files + if filepath.startswith("frontend/"): + return "FRONTEND" + + # Check for plugin files + if filepath.startswith("plugins/"): + return "PLUGINS" + + # Check for backend files + if ( + filepath.startswith("backend/") + or filepath.startswith("testing/backend/") + or filepath == "pyproject.toml" + or ( + filepath.startswith("scripts/") + and not filepath.endswith("check-artifacts.sh") + ) + ): + return "BACKEND" + + # Any other files (root scripts, github workflows, config files) + # UNSAFE to skip: these files affect CI behavior and shared configuration + return "SHARED_OR_CONFIG" + + +def select_tests(files, event_name="push"): + """ + Decides which test suites to run based on changed files and event type. + + Args: + files: List of changed file paths + event_name: GitHub event type ('pull_request' or 'push') + + Returns: + Tuple of (run_backend: bool, run_frontend: bool) + + Logic: + ------ + For PULL REQUESTS (PR checks are required for merge): + - Always run full suite to ensure required checks pass + - This prevents required checks from being marked "skipped" in branch protection + - PR must be thoroughly tested before merge + + For PUSH events (push checks are informational): + - Use selective skipping to save CI time on main/develop + - Skip tests for docs-only changes + - Still run full suite for shared config changes + """ + if not files: + # Empty file list: fall back to running full suite to be safe + return True, True + + # CRITICAL: For pull requests, always run full suite + # This ensures branch protection required checks pass (not skipped) + if event_name == "pull_request": + return True, True + + # For push events, use selective skipping to optimize CI time + categories = {classify_file(f) for f in files} + + # If any changed file is SHARED_OR_CONFIG, run full suite + # These files affect CI behavior and must be thoroughly tested + if "SHARED_OR_CONFIG" in categories: + return True, True + + # If there are both BACKEND and FRONTEND changes, run full suite + if "BACKEND" in categories and "FRONTEND" in categories: + return True, True + + run_backend = False + run_frontend = False + + # If BACKEND or PLUGINS changed, run backend tests + if "BACKEND" in categories or "PLUGINS" in categories: + run_backend = True + + # If FRONTEND changed, run frontend tests + if "FRONTEND" in categories: + run_frontend = True + + return run_backend, run_frontend + + +def write_outputs(run_backend, run_frontend): + """ + Writes GITHUB_OUTPUT variables if the file exists, or prints to stdout. + """ + output_file = os.environ.get("GITHUB_OUTPUT") + backend_str = "true" if run_backend else "false" + frontend_str = "true" if run_frontend else "false" + + if output_file: + with open(output_file, "a") as f: + f.write(f"run_backend={backend_str}\n") + f.write(f"run_frontend={frontend_str}\n") + print( + f"Written to GITHUB_OUTPUT: run_backend={backend_str}, run_frontend={frontend_str}" + ) + else: + print(f"run_backend={backend_str}") + print(f"run_frontend={frontend_str}") + + +def main(): + parser = argparse.ArgumentParser( + description="Determine which tests to run based on changed files and event type." + ) + parser.add_argument( + "--files", + nargs="*", + help="List of changed files. If not specified, git diff will be used to detect changes.", + ) + parser.add_argument( + "--event-name", + default=None, + help="GitHub event name (pull_request or push). If not specified, reads from GITHUB_EVENT_NAME env var.", + ) + args = parser.parse_args() + + if args.files is not None: + files = args.files + else: + files = get_changed_files() + print(f"Detected changed files: {files}") + + event_name = args.event_name or get_event_name() + print(f"Event type: {event_name}") + + run_backend, run_frontend = select_tests(files, event_name=event_name) + write_outputs(run_backend, run_frontend) + + +if __name__ == "__main__": + main() diff --git a/testing/backend/unit/test_select_tests.py b/testing/backend/unit/test_select_tests.py new file mode 100644 index 00000000..e8d4db8b --- /dev/null +++ b/testing/backend/unit/test_select_tests.py @@ -0,0 +1,132 @@ +import os +import sys + +# Add root directory to sys.path so we can import from scripts +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")) +) + +from scripts.select_tests import classify_file, select_tests + + +def test_classify_file_backend(): + assert classify_file("backend/secuscan/main.py") == "BACKEND" + assert classify_file("testing/backend/unit/test_select_tests.py") == "BACKEND" + assert classify_file("pyproject.toml") == "BACKEND" + assert classify_file("scripts/refresh_plugin_checksum.py") == "BACKEND" + + +def test_classify_file_frontend(): + assert classify_file("frontend/src/App.tsx") == "FRONTEND" + assert classify_file("frontend/package.json") == "FRONTEND" + + +def test_classify_file_plugins(): + assert classify_file("plugins/nmap/metadata.json") == "PLUGINS" + assert classify_file("plugins/zap/parser.py") == "PLUGINS" + + +def test_classify_file_docs(): + assert classify_file("README.md") == "DOCS" + assert classify_file("docs/architecture.md") == "DOCS" + assert classify_file("backend/README.md") == "DOCS" + + +def test_classify_file_shared_or_config(): + assert classify_file(".github/workflows/ci.yml") == "SHARED_OR_CONFIG" + assert classify_file(".gitignore") == "SHARED_OR_CONFIG" + assert classify_file("setup.sh") == "SHARED_OR_CONFIG" + assert classify_file("docker-compose.yml") == "SHARED_OR_CONFIG" + assert classify_file("scripts/check-artifacts.sh") == "SHARED_OR_CONFIG" + + +def test_select_tests_empty(): + # Empty changed file list must fall back to running everything + assert select_tests([]) == (True, True) + + +def test_select_tests_backend_only(): + assert select_tests(["backend/secuscan/main.py"]) == (True, False) + assert select_tests( + ["backend/secuscan/main.py", "testing/backend/unit/test_models.py"] + ) == (True, False) + + +def test_select_tests_frontend_only(): + assert select_tests(["frontend/src/App.tsx"]) == (False, True) + assert select_tests(["frontend/src/App.tsx", "frontend/package.json"]) == ( + False, + True, + ) + + +def test_select_tests_plugins_only(): + # Plugins only should run backend tests + assert select_tests(["plugins/nmap/metadata.json"]) == (True, False) + + +def test_select_tests_docs_only(): + # Docs only should run no code tests + assert select_tests(["README.md"]) == (False, False) + assert select_tests(["README.md", "docs/architecture.md"]) == (False, False) + + +def test_select_tests_mixed_backend_frontend(): + # Mixed backend and frontend changes should run everything + assert select_tests(["backend/secuscan/main.py", "frontend/src/App.tsx"]) == ( + True, + True, + ) + + +def test_select_tests_mixed_backend_plugins(): + # Backend + plugins should only run backend tests + assert select_tests(["backend/secuscan/main.py", "plugins/nmap/metadata.json"]) == ( + True, + False, + ) + + +def test_select_tests_mixed_frontend_docs(): + # Frontend + docs should only run frontend tests + assert select_tests(["frontend/src/App.tsx", "README.md"]) == (False, True) + + +def test_select_tests_shared_config_fallback(): + # Any shared config file triggers full suite fallback + assert select_tests([".github/workflows/ci.yml"]) == (True, True) + assert select_tests(["setup.sh", "backend/secuscan/main.py"]) == (True, True) + assert select_tests([".gitignore", "frontend/src/App.tsx"]) == (True, True) + + +# ── Event-based logic (PR vs push) ──────────────────────────────────────────── + +def test_select_tests_pull_request_always_full_suite(): + """ + PRs must always run full suite to ensure required branch protection checks + do not get marked as 'skipped'. + """ + # Docs-only PR: still run full suite for safety + assert select_tests(["README.md"], event_name="pull_request") == (True, True) + # Backend-only PR: full suite (always) + assert select_tests(["backend/secuscan/main.py"], event_name="pull_request") == ( + True, + True, + ) + # Frontend-only PR: full suite (always) + assert select_tests(["frontend/src/App.tsx"], event_name="pull_request") == ( + True, + True, + ) + + +def test_select_tests_push_uses_selective_skipping(): + """ + Push events (to main) can use selective skipping to optimize CI time. + """ + # Push with docs-only: skip tests + assert select_tests(["README.md"], event_name="push") == (False, False) + # Push with backend: run backend only + assert select_tests(["backend/secuscan/main.py"], event_name="push") == (True, False) + # Push with frontend: run frontend only + assert select_tests(["frontend/src/App.tsx"], event_name="push") == (False, True)