feat(ci): add changed-file scoped test selection with full-suite fallback (#243)

.github/workflows/ci.yml

@@ -8,7 +8,44 @@ permissions:
   contents: read
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      run_backend: ${{ steps.filter.outputs.run_backend }}
+      run_frontend: ${{ steps.filter.outputs.run_frontend }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Fetch base branch for diff
+        if: github.event_name == 'pull_request'
+        run: git fetch origin "${{ github.base_ref }}" --depth=1
+      - name: Determine test selection
+        id: filter
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+        run: python3 scripts/select_tests.py
+
+  formatting-hygiene:
+    needs: detect-changes
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Check formatting hygiene on changed files
+        run: |
+          git fetch origin "${{ github.base_ref }}" --depth=1
+          git diff --check "origin/${{ github.base_ref }}"...HEAD
+
   backend-lint:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.run_backend == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -25,6 +62,12 @@ jobs:
         run: ruff check backend testing/backend
 
   backend-tests:
+    needs: [detect-changes, backend-lint, formatting-hygiene]
+    if: |
+      always() &&
+      needs.detect-changes.outputs.run_backend == 'true' &&
+      (needs.backend-lint.result == 'success' || needs.backend-lint.result == 'skipped') &&
+      (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -69,6 +112,11 @@ jobs:
           echo "::warning::Performance benchmark thresholds exceeded or benchmarks failed to run. Check the job logs for details."
 
   frontend-checks:
+    needs: [detect-changes, formatting-hygiene]
+    if: |
+      always() &&
+      needs.detect-changes.outputs.run_frontend == 'true' &&
+      (needs.formatting-hygiene.result == 'success' || needs.formatting-hygiene.result == 'skipped')
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -94,15 +142,3 @@ jobs:
         run: npm run test
       - name: Build frontend
         run: npm run build
-
-  formatting-hygiene:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Check formatting hygiene on changed files
-        run: |
-          git fetch origin "${{ github.base_ref }}" --depth=1
-          git diff --check "origin/${{ github.base_ref }}"...HEAD

docs/ci-test-selection.md

@@ -0,0 +1,110 @@
+## CI Test Selection (Changed-file scoped)
+
+Purpose
+-------
+This document explains the changed-file scoped CI test selection implemented in `scripts/select_tests.py`.
+
+## Branch Protection Safety Guarantee
+
+**CRITICAL:** This implementation is safe for use with GitHub branch protection because:
+
+1. **Pull Requests Always Run Full Suite**
+   - All PR changes ALWAYS trigger the full test suite (backend + frontend)
+   - This ensures required checks configured in branch protection never get marked as "skipped"
+   - A skipped required check would incorrectly block the PR merge
+   - Example: A docs-only PR still runs all tests to satisfy required checks
+
+2. **Push Events Use Selective Skipping**
+   - Only push events (commits to main/develop) use selective skipping
+   - Push events are informational and don't block merges
+   - Selective skipping saves CI time on post-merge verification
+   - Example: A docs-only commit to main skips tests safely (no blocking rules)
+
+## Event Type Behavior
+
+| Event        | Docs-Only | Backend Only | Frontend Only | Mixed / Config  |
+|--------------|-----------|--------------|---------------|-----------------|
+| `pull_request` | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite | ✓ Full Suite    |
+| `push`       | ✗ Skip All | ✓ Backend    | ✓ Frontend    | ✓ Full Suite    |
+
+Legend: ✓ = runs tests, ✗ = skips tests
+
+## File Classification
+
+The script classifies changed files into these categories:
+
+- **DOCS** (skippable): `.md` files anywhere, `docs/` directory
+  - Safe to skip because docs cannot affect code behavior
+  - Exception: In PRs, still runs full suite for branch protection
+
+- **FRONTEND**: `frontend/` directory
+  - Runs frontend tests only (unless mixed with backend)
+
+- **BACKEND**: `backend/`, `testing/backend/`, `pyproject.toml`, `scripts/*.py`
+  - Runs backend tests; includes plugins via plugin dependencies
+
+- **PLUGINS**: `plugins/` directory
+  - Treated as backend changes (runs backend tests)
+
+- **SHARED_OR_CONFIG** (forces full suite): `.github/`, root scripts, config files
+  - `.github/workflows/` changes → full suite (CI behavior changes)
+  - `setup.sh`, `docker-compose.yml` → full suite (system changes)
+  - These affect multiple subsystems and must be fully tested
+
+## Why This is Safe
+
+1. **Conservative Fallbacks**
+   - When in doubt, we run the full suite
+   - Docs-only + shared config → full suite
+   - Backend + frontend → full suite
+   - Mixed categories → full suite
+
+2. **Branch Protection Guaranteed**
+   - PRs cannot skip required checks (always full suite)
+   - Developers cannot accidentally merge untested code
+   - Required checks will pass/fail, never be skipped
+
+3. **Deterministic Classification**
+   - File paths are mapped consistently
+   - No heuristics or guessing
+   - Can be verified locally
+
+## Configuration
+
+To modify the test selection policy:
+
+1. Update file classification in `scripts/select_tests.py` → `classify_file()`
+2. Update logic in `select_tests()` function
+3. Add corresponding unit tests in `testing/backend/unit/test_select_tests.py`
+4. Run tests locally: `pytest testing/backend/unit/test_select_tests.py -v`
+5. Update this document if behavior changes
+
+## Local Testing
+
+Dry-run the selection tool locally:
+
+```bash
+# Test with specific files (simulating a PR/push)
+python3 scripts/select_tests.py --files backend/main.py frontend/App.tsx --event-name pull_request
+# Output: run_backend=true, run_frontend=true (PR always runs full suite)
+
+python3 scripts/select_tests.py --files README.md --event-name push
+# Output: run_backend=false, run_frontend=false (docs-only push skips tests)
+
+python3 scripts/select_tests.py --files .github/workflows/ci.yml
+# Output: run_backend=true, run_frontend=true (config changes run full suite)
+```
+
+## Required Checks in GitHub
+
+For this to work correctly, configure your branch protection rule on `main` to require these checks:
+
+- `formatting-hygiene` (always runs, PR-only)
+- `backend-lint` (skippable based on changes)
+- `backend-tests` (skippable based on changes)
+- `frontend-checks` (skippable based on changes)
+
+This allows:
+- ✅ Docs-only PR → required checks (formatting) run, optional checks (tests) run full suite
+- ✅ Backend-only PR → full suite runs, satisfying all required checks
+- ✅ Docs-only push → tests skip safely (push checks are not required)

scripts/select_tests.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+import subprocess
+
+
+def get_changed_files():
+    """
+    Attempts to detect changed files using git diff.
+    """
+    base_ref = os.environ.get("GITHUB_BASE_REF", "main")
+    # Commands to try in order of preference
+    commands = [
+        ["git", "diff", "--name-only", f"origin/{base_ref}...HEAD"],
+        ["git", "diff", "--name-only", f"{base_ref}...HEAD"],
+        ["git", "diff", "--name-only", "HEAD~1"],
+        ["git", "diff", "--name-only"],
+    ]
+    for cmd in commands:
+        try:
+            res = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            files = [line.strip() for line in res.stdout.splitlines() if line.strip()]
+            if files:
+                return files
+        except Exception:
+            continue
+    return []
+
+
+def get_event_name():
+    """
+    Get the GitHub event name (push or pull_request).
+    Defaults to 'push' for local/unknown environments.
+    """
+    return os.environ.get("GITHUB_EVENT_NAME", "push")
+
+
+def classify_file(filepath):
+    """
+    Classifies a file path into a logical CI category.
+
+    Categories:
+    - DOCS: Documentation files (.md) - safe to skip for selective testing
+    - FRONTEND: Frontend code and tests
+    - PLUGINS: Plugin definitions
+    - BACKEND: Backend code, tests, and Python configs
+    - SHARED_OR_CONFIG: Shared configuration and CI workflow files - always requires full suite
+    """
+    filepath = filepath.strip()
+    if not filepath:
+        return "DOCS"
+
+    # Convert backslashes to forward slashes for cross-platform robustness
+    filepath = filepath.replace("\\", "/")
+
+    # Check for docs first (both .md files anywhere and anything in docs/ directory)
+    # SAFE to skip: documentation cannot affect code behavior
+    if filepath.endswith(".md") or filepath.startswith("docs/"):
+        return "DOCS"
+
+    # Check for frontend files
+    if filepath.startswith("frontend/"):
+        return "FRONTEND"
+
+    # Check for plugin files
+    if filepath.startswith("plugins/"):
+        return "PLUGINS"
+
+    # Check for backend files
+    if (
+        filepath.startswith("backend/")
+        or filepath.startswith("testing/backend/")
+        or filepath == "pyproject.toml"
+        or (
+            filepath.startswith("scripts/")
+            and not filepath.endswith("check-artifacts.sh")
+        )
+    ):
+        return "BACKEND"
+
+    # Any other files (root scripts, github workflows, config files)
+    # UNSAFE to skip: these files affect CI behavior and shared configuration
+    return "SHARED_OR_CONFIG"
+
+
+def select_tests(files, event_name="push"):
+    """
+    Decides which test suites to run based on changed files and event type.
+
+    Args:
+        files: List of changed file paths
+        event_name: GitHub event type ('pull_request' or 'push')
+
+    Returns:
+        Tuple of (run_backend: bool, run_frontend: bool)
+
+    Logic:
+    ------
+    For PULL REQUESTS (PR checks are required for merge):
+        - Always run full suite to ensure required checks pass
+        - This prevents required checks from being marked "skipped" in branch protection
+        - PR must be thoroughly tested before merge
+
+    For PUSH events (push checks are informational):
+        - Use selective skipping to save CI time on main/develop
+        - Skip tests for docs-only changes
+        - Still run full suite for shared config changes
+    """
+    if not files:
+        # Empty file list: fall back to running full suite to be safe
+        return True, True
+
+    # CRITICAL: For pull requests, always run full suite
+    # This ensures branch protection required checks pass (not skipped)
+    if event_name == "pull_request":
+        return True, True
+
+    # For push events, use selective skipping to optimize CI time
+    categories = {classify_file(f) for f in files}
+
+    # If any changed file is SHARED_OR_CONFIG, run full suite
+    # These files affect CI behavior and must be thoroughly tested
+    if "SHARED_OR_CONFIG" in categories:
+        return True, True
+
+    # If there are both BACKEND and FRONTEND changes, run full suite
+    if "BACKEND" in categories and "FRONTEND" in categories:
+        return True, True
+
+    run_backend = False
+    run_frontend = False
+
+    # If BACKEND or PLUGINS changed, run backend tests
+    if "BACKEND" in categories or "PLUGINS" in categories:
+        run_backend = True
+
+    # If FRONTEND changed, run frontend tests
+    if "FRONTEND" in categories:
+        run_frontend = True
+
+    return run_backend, run_frontend
+
+
+def write_outputs(run_backend, run_frontend):
+    """
+    Writes GITHUB_OUTPUT variables if the file exists, or prints to stdout.
+    """
+    output_file = os.environ.get("GITHUB_OUTPUT")
+    backend_str = "true" if run_backend else "false"
+    frontend_str = "true" if run_frontend else "false"
+
+    if output_file:
+        with open(output_file, "a") as f:
+            f.write(f"run_backend={backend_str}\n")
+            f.write(f"run_frontend={frontend_str}\n")
+        print(
+            f"Written to GITHUB_OUTPUT: run_backend={backend_str}, run_frontend={frontend_str}"
+        )
+    else:
+        print(f"run_backend={backend_str}")
+        print(f"run_frontend={frontend_str}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Determine which tests to run based on changed files and event type."
+    )
+    parser.add_argument(
+        "--files",
+        nargs="*",
+        help="List of changed files. If not specified, git diff will be used to detect changes.",
+    )
+    parser.add_argument(
+        "--event-name",
+        default=None,
+        help="GitHub event name (pull_request or push). If not specified, reads from GITHUB_EVENT_NAME env var.",
+    )
+    args = parser.parse_args()
+
+    if args.files is not None:
+        files = args.files
+    else:
+        files = get_changed_files()
+        print(f"Detected changed files: {files}")
+
+    event_name = args.event_name or get_event_name()
+    print(f"Event type: {event_name}")
+
+    run_backend, run_frontend = select_tests(files, event_name=event_name)
+    write_outputs(run_backend, run_frontend)
+
+
+if __name__ == "__main__":
+    main()