From a6d75836d270ba55820612e245ea4b7512e8126b Mon Sep 17 00:00:00 2001 From: "David J. Bianco" Date: Fri, 15 May 2026 12:56:20 -0400 Subject: [PATCH] chore: split slow tests from coverage --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++++++++++++--- AGENTS.md | 2 +- CONTRIBUTING.md | 21 ++++++++++++--------- README.md | 7 +++++-- TODO.md | 1 + pyproject.toml | 1 - 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ef42a91..e07530a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,13 @@ jobs: - name: Install dependencies run: uv sync --all-extras - - name: Run tests with coverage - if: github.ref_name != 'dev' && github.base_ref != 'dev' - run: uv run pytest --include-slow --cov-report=xml + - name: Run tests with coverage (Python 3.12) + if: matrix.python-version == '3.12' && github.ref_name != 'dev' && github.base_ref != 'dev' + run: uv run pytest --cov-report=xml + + - name: Run compatibility tests (Python 3.11) + if: matrix.python-version == '3.11' && github.ref_name != 'dev' && github.base_ref != 'dev' + run: uv run pytest --no-cov - name: Run fast unit tests (dev) if: github.ref_name == 'dev' || github.base_ref == 'dev' @@ -48,6 +52,32 @@ jobs: file: ./coverage.xml fail_ci_if_error: false + slow-comprehensive: + name: Slow comprehensive tests + runs-on: ubuntu-latest + timeout-minutes: 20 + if: github.ref_name != 'dev' && github.base_ref != 'dev' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --all-extras + + - name: Run slow comprehensive tests without coverage + run: uv run pytest --include-slow -m slow --no-cov --durations=20 + lint: name: Lint runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 9a42fe73..60cd18a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ When a phase is fully complete, collapse its tasks in `TODO.md` to a 2-3 line su **Testing:** - pytest with pytest-cov, pytest-asyncio, pytest-mock, pytest-benchmark -- Separate test markers: `@pytest.mark.slow` for large dataset tests (not run by default) +- Separate test markers: `@pytest.mark.slow` for large dataset/workload tests (not run by default). Run slow tests with `--no-cov` unless you are specifically profiling coverage behavior, because coverage instrumentation makes the generator workload much slower. - Target coverage: 95%+ overall, 95%+ for core generation engine **Format Support:** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf7a9563..8b65dd88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,11 +46,14 @@ We expect new pull requests to include tests for any affected behavior, and, as we follow semantic versioning, we may reserve breaking changes until the next major version release. -Before submitting, run the full test suite (including slow tests) and confirm -all tests pass: +Before submitting, run the normal coverage-gated suite, the slow comprehensive +suite without coverage instrumentation, and lint/format checks: ```bash -uv run pytest --include-slow +uv run pytest +uv run pytest --include-slow -m slow --no-cov --durations=20 +uv run ruff check . +uv run ruff format --check . ``` ### Commit Messages @@ -93,18 +96,18 @@ uv sync uv run pytest # Lint and format -uv run ruff check src/ tests/ -uv run ruff format src/ tests/ +uv run ruff check . +uv run ruff format --check . ``` ### Test Markers -- `@pytest.mark.slow`: large dataset tests (100+ users), skipped by default +- `@pytest.mark.slow`: large dataset and workload tests, skipped by default and normally + run without coverage instrumentation ```bash -uv run pytest # Quick run (skips slow tests) -uv run pytest --include-slow # Full run (all tests, required before PRs) -uv run pytest -m slow # Only slow tests +uv run pytest # Normal coverage-gated run +uv run pytest --include-slow -m slow --no-cov --durations=20 # Slow comprehensive run ``` ## Code Style diff --git a/README.md b/README.md index 95a5dd37..d8770ae5 100644 --- a/README.md +++ b/README.md @@ -244,12 +244,15 @@ uv sync # Run tests (1400+ tests) uv run pytest +# Run slow comprehensive workload tests without coverage instrumentation +uv run pytest --include-slow -m slow --no-cov --durations=20 + # Run specific test suite uv run pytest tests/unit/test_network_visibility.py -v # Lint and format -uv run ruff check src/ tests/ -uv run ruff format src/ tests/ +uv run ruff check . +uv run ruff format --check . ``` ### Tech Stack diff --git a/TODO.md b/TODO.md index 6be70cb9..60f16345 100644 --- a/TODO.md +++ b/TODO.md @@ -36,6 +36,7 @@ Replaced manual per-emitter field coordination with SecurityEvent intermediate r ## Pre-MVP: Consolidated Quality Fixes — IN PROGRESS +- [x] Split slow comprehensive tests from coverage instrumentation in CI and update contributor/agent testing guidance — normal coverage gate passed at 79.38% with slow tests skipped; slow comprehensive suite passed separately with `--no-cov` in 2m36s; Ruff checks passed. - [x] Prepare `dev` → `main` PR — inspected `main..dev`, applied the required v0.6.2 version/changelog bump on `dev`, ran release checks, pushed, and opened the PR into `main`. - [x] Retargeted and merged PR fixes #138-#141 into `dev`, then reworked PR #137 against current `dev` so Windows event spool hardening preserves all current emitter fixup passes before merge. - [x] Remediate Windows singleton PID path traversal telemetry suppression — canonicalize Windows singleton paths with ntpath before seeded PID reuse and cover traversal variants with a unit test. diff --git a/pyproject.toml b/pyproject.toml index f7ec65ec..229ded2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,6 @@ addopts = [ "--tb=short", "--cov=evidenceforge", "--cov-report=term-missing", - "--cov-report=html", ] filterwarnings = [ "error",