diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01bc2d96..40ad08a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-report - path: implementations/python/coverage.xml + path: | + implementations/python/coverage.xml + implementations/python/coverage.json fuzz: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f599c3f7..839fee2c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +coverage.json *.cover *.py.cover .hypothesis/ diff --git a/Makefile b/Makefile index fa33532a..e18c6a57 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ .PHONY: policy policy: - uv tool run --from 'nox[uv]==2026.4.10' nox -f noxfile.py -s policy + uv tool run --from 'nox[uv]==2026.4.10' nox -f noxfile.py -s policy $(if $(strip $(RAES_REQUIREMENT_UID)),,-- --skip-requirement) diff --git a/docs/decisions/adrs/README.md b/docs/decisions/adrs/README.md index d065ab01..405abefb 100644 --- a/docs/decisions/adrs/README.md +++ b/docs/decisions/adrs/README.md @@ -251,3 +251,4 @@ adr-101-adversarial-participant-flow-control | [100](adr-100-participant-crossing-bisimulation.md) | Proof-Bearing Participant-Crossing Bisimulation | accepted | 2026-07-29 | | [101](adr-101-adversarial-participant-flow-control.md) | Adversarial Participant Boundary Flow Control | accepted | 2026-07-30 | | [102](adr-102-mixed-cross-backend-participant-control.md) | Mixed Cross-Backend Participant Control | accepted | 2026-07-31 | +| [103](adr-103-branch-aware-python-coverage-policy.md) | Branch-Aware Python Coverage Policy | accepted | 2026-08-13 | diff --git a/docs/decisions/adrs/adr-103-branch-aware-python-coverage-policy.md b/docs/decisions/adrs/adr-103-branch-aware-python-coverage-policy.md new file mode 100644 index 00000000..320a288f --- /dev/null +++ b/docs/decisions/adrs/adr-103-branch-aware-python-coverage-policy.md @@ -0,0 +1,105 @@ +# ADR-103: Branch-Aware Python Coverage Policy + +## Status + +accepted + +## Date + +2026-08-13 + +## Classification + +Classification: FM0 +Required artifacts: repository configuration and focused policy tests +Waivers: none + +## Context + +ADR-014 makes nox the canonical verification graph and already places Python +coverage collection in that graph. The coverage configuration measures +statements only, omits some repository-owned Python entry points, and accepts a +50% aggregate floor even though the integrated repository is close to 90% line +coverage. Branch outcomes are therefore invisible and the threshold does not +represent the repository's quality policy. + +Coverage is useful as a regression guard and review signal, but it is not a +proof of behavioral correctness. Requiring 100% aggregate or changed-code +coverage creates incentives to execute lines without asserting behavior, +over-mock difficult boundaries, distort defensive code, or add exclusions that +serve the metric rather than the design. + +ADR-015 also defines the trust model for repository-authored policy: gates +prevent accidental regressions, while review protects the mutable gate itself. +An in-repository coverage checker cannot provide an adversarial security +boundary because the checker, its configuration, tests, and invocation can all +change in the same pull request. + +## Decision + +1. Coverage.py remains the authority for Python statement, exclusion, and + branch semantics. The repository does not maintain a second Python semantic + analyzer for coverage. +2. Canonical unit and integration coverage is combined before reporting. + Coverage collection includes shipped packages, repository tooling, + `noxfile.py`, and the Hatch build hook. Tests, documentation, generated + environments, and acquired caches remain outside the production denominator. +3. Canonical verification enforces a fixed 90% aggregate **line-coverage** + floor. The floor is an explicit quality threshold, not a manually advanced + ratchet. Falling below it fails verification; exceeding it does not require a + metadata update. +4. Canonical verification measures branch coverage and publishes it in XML and + JSON reports. This ADR establishes no aggregate or changed-branch threshold; + branch results remain visible for review until repository evidence justifies + a separate threshold. +5. The repository does not impose blanket 100% changed-line or changed-branch + coverage. Reviewers apply the proportionate FM0-FM3 assurance policy from + ADR-007 and ADR-018 when a change needs stronger evidence than the aggregate + floor. +6. Coverage configuration and report handling use ordinary fail-closed input + checks for malformed or missing data. They do not attempt to resist a pull + request author who can edit the gate itself; that would require a separately + accepted immutable-CI trust boundary. +7. Coverage.py XML remains the SonarCloud input. JSON is published alongside it + so the line-only threshold and branch totals are inspectable without + reinterpreting Python source. + +## Alternatives Considered + +### A manually maintained aggregate ratchet + +Rejected. A floor that can only be raised still permits all gains above a stale +recorded value to be lost. It depends on recurring manual updates while +presenting itself as automatic regression protection. + +### 100% coverage for changed code + +Rejected. Diff boundaries are a poor proxy for behavioral risk, particularly +for small changes, defensive paths, multiline statements, and branch exits. +Enforcing the rule also requires semantic diff machinery disproportionate to +the repository's accidental-regression trust model. + +### Exact base-versus-head aggregate comparison + +Not adopted. Running the complete unit and integration suite at both revisions +would provide an automatic debt comparison, but it would approximately double +the coverage portion of canonical verification. The fixed floor and branch +visibility provide the intended guardrail at materially lower cost. + +### SonarCloud as the only coverage gate + +Rejected. SonarCloud provides useful new-code line and condition metrics, but it +is an external service and is unavailable to some pull-request contexts. The +canonical local graph must retain its repository-owned aggregate floor. + +## Consequences + +- Coverage reports include branch information and all agreed repository-owned + Python surfaces. +- The coverage policy is understandable from standard Coverage.py behavior and + a small line-total check rather than a custom semantic analyzer. +- The 90% floor remains stable and meaningful without maintenance commits. +- Branch coverage may regress without independently failing while no branch + threshold is adopted; the reports and SonarCloud make that movement visible. +- Changes that intentionally alter measured scope must update this ADR or be + justified by a later superseding decision. diff --git a/docs/decisions/adrs/adr-index.yaml b/docs/decisions/adrs/adr-index.yaml index efcc6505..9b47fc56 100644 --- a/docs/decisions/adrs/adr-index.yaml +++ b/docs/decisions/adrs/adr-index.yaml @@ -500,3 +500,6 @@ adrs: - id: ADR-102 path: docs/decisions/adrs/adr-102-mixed-cross-backend-participant-control.md pin: cfcb0f5b7291b47fe6ff6096bb18dd9063b83ee5d4f7f6a9cafbdd06f21bf716 + - id: ADR-103 + path: docs/decisions/adrs/adr-103-branch-aware-python-coverage-policy.md + pin: a44acbc2db1b5349ba316ba0d09bd9b46310db016cb87937b08bc6bc107755aa diff --git a/implementations/python/pyproject.toml b/implementations/python/pyproject.toml index 788f05f5..5f4ba06c 100644 --- a/implementations/python/pyproject.toml +++ b/implementations/python/pyproject.toml @@ -123,29 +123,19 @@ addopts = "-m 'not fuzz and not integration and not docker'" timeout = 120 [tool.coverage.run] -source = [ - "raes_contracts", - "raes_backend_protocols", - "raes_backend_stubs", - "raes_backend_libvirt", - "raes_operations", - "raes_reference_backend", - "raes_cli", - "raes_conformance", - "raes_mcp", - "raes_processor", - "raes_runtime", - "raes", - # Repo CI tooling (issue #54): the `tools/` tree lives at the repo root and - # is imported as the top-level `tools` package via pytest's `pythonpath` - # (`../..`). Coverage matches source packages by module name, so its - # `tools.*` modules are measured even though pytest runs from - # `implementations/python/`. - "tools", +branch = true +relative_files = false +source = ["../.."] +omit = [ + "*/.cache/*", + "*/docs/*", + "*/implementations/python/.venv/*", + "*/implementations/python/tests/*", ] [tool.coverage.report] show_missing = true +include_namespace_packages = true exclude_also = [ # The reference backend's OCI subprocess leaf (RUN-314): real container # IO, exercised only by the opt-in docker integration tests, never in the diff --git a/implementations/python/tests/test_repo_policy_tools.py b/implementations/python/tests/test_repo_policy_tools.py index e1c40e0d..96e2dcc2 100644 --- a/implementations/python/tests/test_repo_policy_tools.py +++ b/implementations/python/tests/test_repo_policy_tools.py @@ -124,6 +124,7 @@ def chdir(self, _path: Path): assert kwargs["env"] == {"COVERAGE_FILE": str(coverage_file)} session.commands.clear() + monkeypatch.setattr(noxfile, "_enforce_line_coverage", lambda _path: 90.0) noxfile._run_pytest( session, "-m", @@ -143,8 +144,8 @@ def chdir(self, _path: Path): for command, options in session.commands if command[:4] == ("uv", "run", "--frozen", "coverage") ] - assert [command[4] for command, _options in coverage_commands] == ["xml", "report"] - assert coverage_commands[-1][0][-2:] == ("--fail-under=50", "--format=total") + assert [command[4] for command, _options in coverage_commands] == ["xml", "json", "report"] + assert coverage_commands[-1][0][-1:] == ("--format=total",) assert all(options["env"] == {"COVERAGE_FILE": str(coverage_file)} for _, options in coverage_commands) @@ -299,6 +300,7 @@ def chdir(self, _path: Path): return nullcontext() session = FakeSession() + monkeypatch.setattr(noxfile, "_enforce_line_coverage", lambda _path: 90.0) noxfile._finalize_parallel_coverage(session, tmp_path) coverage_commands = [ @@ -306,14 +308,90 @@ def chdir(self, _path: Path): for command, options in session.commands if command[:4] == ("uv", "run", "--frozen", "coverage") ] - assert [command[4] for command, _options in coverage_commands] == ["combine", "xml", "report"] + assert [command[4] for command, _options in coverage_commands] == ["combine", "xml", "json", "report"] assert coverage_commands[0][0][5:] == ("--keep", str(tmp_path)) - assert coverage_commands[-1][0][-2:] == ("--fail-under=50", "--format=total") + assert coverage_commands[-1][0][-1:] == ("--format=total",) assert all( options["env"] == {"COVERAGE_FILE": str(tmp_path / ".coverage")} for _command, options in coverage_commands ) +def test_coverage_configuration_measures_branches_and_repository_python() -> None: + config = tomllib.loads((REPO_ROOT / "implementations" / "python" / "pyproject.toml").read_text(encoding="utf-8")) + run = config["tool"]["coverage"]["run"] + report = config["tool"]["coverage"]["report"] + + assert run["branch"] is True + assert run["source"] == ["../.."] + assert set(run["omit"]) == { + "*/.cache/*", + "*/docs/*", + "*/implementations/python/.venv/*", + "*/implementations/python/tests/*", + } + assert report["include_namespace_packages"] is True + + +def test_line_coverage_threshold_is_fixed_at_ninety_percent( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + noxfile = load_noxfile_with_fake_nox(monkeypatch) + report_path = tmp_path / "coverage.json" + report_path.write_text( + json.dumps({"totals": {"covered_lines": 90, "num_statements": 100}}), + encoding="utf-8", + ) + + assert noxfile._enforce_line_coverage(report_path) == 90.0 + + report_path.write_text( + json.dumps({"totals": {"covered_lines": 899, "num_statements": 1000}}), + encoding="utf-8", + ) + with pytest.raises(RuntimeError, match="89.900% is below required 90.000%"): + noxfile._enforce_line_coverage(report_path) + + +def test_make_policy_skips_only_requirement_governance_without_a_uid() -> None: + environment = os.environ.copy() + environment.pop("RAES_REQUIREMENT_UID", None) + requirement_free = subprocess.run( + ("make", "--dry-run", "policy"), + cwd=REPO_ROOT, + env=environment, + check=True, + capture_output=True, + text=True, + ).stdout + environment["RAES_REQUIREMENT_UID"] = "ASR-505" + requirement_scoped = subprocess.run( + ("make", "--dry-run", "policy"), + cwd=REPO_ROOT, + env=environment, + check=True, + capture_output=True, + text=True, + ).stdout + + assert requirement_free.rstrip().endswith("-- --skip-requirement") + assert "--skip-requirement" not in requirement_scoped + + +def test_hook_policy_context_skips_only_requirement_free_branches(monkeypatch: pytest.MonkeyPatch) -> None: + noxfile = load_noxfile_with_fake_nox(monkeypatch) + monkeypatch.delenv("RAES_REQUIREMENT_UID", raising=False) + monkeypatch.setattr(noxfile, "_git_lines", lambda *_args: ["1104-minimal-coverage-policy"]) + assert noxfile._requirement_aware_policy_args("--staged") == ["--staged", "--skip-requirement"] + + monkeypatch.setattr(noxfile, "_git_lines", lambda *_args: ["1104-ASR-505-coverage-policy"]) + assert noxfile._requirement_aware_policy_args("--staged") == ["--staged"] + + monkeypatch.setenv("RAES_REQUIREMENT_UID", "ASR-505") + monkeypatch.setattr(noxfile, "_git_lines", lambda *_args: ["1104-minimal-coverage-policy"]) + assert noxfile._requirement_aware_policy_args("--staged") == ["--staged"] + + def test_docs_graph_uses_curated_root_and_reader_style_gate( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/noxfile.py b/noxfile.py index b529b538..6c20da72 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,9 @@ from dataclasses import dataclass from pathlib import Path from time import perf_counter +import json import os +import re import shutil import subprocess import sys @@ -54,6 +56,10 @@ RUFF_CONFIG = PROJECT_ROOT / "pyproject.toml" OSV_LOCKFILE_PATH = PROJECT_ROOT / "uv.lock" OSV_REPORT_PATH = PROJECT_ROOT / "osv-scanner-report.json" +COVERAGE_XML_PATH = PROJECT_ROOT / "coverage.xml" +COVERAGE_JSON_PATH = PROJECT_ROOT / "coverage.json" +MINIMUM_LINE_COVERAGE_PERCENT = 90.0 +REQUIREMENT_UID_RE = re.compile(r"(?:^|[^A-Z0-9])[A-Z]{3}-[0-9]{3}(?:$|[^A-Z0-9])") TARGETED_POLICY_TESTS = [ "implementations/python/tests/test_repo_policy_tools.py", "implementations/python/tests/test_requirement_governance.py", @@ -268,18 +274,7 @@ def _run_pytest( with session.chdir(PROJECT_ROOT): _run(session, *command, env=coverage_env) if finalize_coverage: - _run(session, "uv", "run", "--frozen", "coverage", "xml", env=coverage_env) - _run( - session, - "uv", - "run", - "--frozen", - "coverage", - "report", - "--fail-under=50", - "--format=total", - env=coverage_env, - ) + _write_and_check_coverage(session, coverage_env) def _split_policy_session_args(posargs: list[str]) -> tuple[list[str], list[str], bool]: @@ -308,6 +303,15 @@ def _split_policy_session_args(posargs: list[str]) -> tuple[list[str], list[str] return repo_args, requirement_args, skip_requirement +def _requirement_aware_policy_args(*args: str) -> list[str]: + if os.environ.get("RAES_REQUIREMENT_UID", "").strip(): + return list(args) + branch = next(iter(_git_lines("branch", "--show-current")), "") + if REQUIREMENT_UID_RE.search(branch): + return list(args) + return [*args, "--skip-requirement"] + + def _parse_hygiene_posargs(posargs: Sequence[str], *, default_all_files: bool) -> HygieneSelection: staged = False base_rev: str | None = None @@ -869,6 +873,66 @@ def _run_integration_tests( ) +def _enforce_line_coverage(report_path: Path) -> float: + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + totals = report["totals"] + covered_lines = totals["covered_lines"] + statements = totals["num_statements"] + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as exc: + raise RuntimeError(f"could not read line coverage totals from {report_path}") from exc + if ( + not isinstance(covered_lines, int) + or isinstance(covered_lines, bool) + or not isinstance(statements, int) + or isinstance(statements, bool) + or covered_lines < 0 + or statements < 0 + or covered_lines > statements + ): + raise RuntimeError(f"invalid line coverage totals in {report_path}") + percent = 100.0 if statements == 0 else 100.0 * covered_lines / statements + if percent + 1e-12 < MINIMUM_LINE_COVERAGE_PERCENT: + raise RuntimeError(f"line coverage {percent:.3f}% is below required {MINIMUM_LINE_COVERAGE_PERCENT:.3f}%") + return percent + + +def _write_and_check_coverage(session: nox.Session, coverage_env: dict[str, str]) -> None: + _run( + session, + "uv", + "run", + "--frozen", + "coverage", + "xml", + "-o", + str(COVERAGE_XML_PATH), + env=coverage_env, + ) + _run( + session, + "uv", + "run", + "--frozen", + "coverage", + "json", + "-o", + str(COVERAGE_JSON_PATH), + env=coverage_env, + ) + _run( + session, + "uv", + "run", + "--frozen", + "coverage", + "report", + "--format=total", + env=coverage_env, + ) + _enforce_line_coverage(COVERAGE_JSON_PATH) + + def _finalize_parallel_coverage(session: nox.Session, coverage_dir: Path) -> None: coverage_file = coverage_dir / ".coverage" coverage_env = {"COVERAGE_FILE": str(coverage_file)} @@ -884,18 +948,7 @@ def _finalize_parallel_coverage(session: nox.Session, coverage_dir: Path) -> Non str(coverage_dir), env=coverage_env, ) - _run(session, "uv", "run", "--frozen", "coverage", "xml", env=coverage_env) - _run( - session, - "uv", - "run", - "--frozen", - "coverage", - "report", - "--fail-under=50", - "--format=total", - env=coverage_env, - ) + _write_and_check_coverage(session, coverage_env) def _run_docker_integration_tests(session: nox.Session, reporter: SessionReporter) -> None: @@ -1192,7 +1245,7 @@ def hook_pre_commit(session: nox.Session) -> None: changed_tests = select_changed_python_tests(changed) try: _run_hygiene(session, reporter, posargs=changed, default_all_files=False) - _run_policy(session, reporter, "--staged") + _run_policy(session, reporter, *_requirement_aware_policy_args("--staged")) _run_changed_lint(session, reporter, changed) if _paths_trigger(changed, CONTRACT_TRIGGER_PREFIXES): _run_contracts(session, reporter) @@ -1258,7 +1311,8 @@ def _run_changed_verification( plan = plan_for_changes([]) session.log(f"change classification failed closed to the full local gate: {exc}") - policy_args = ["--base-rev", base_rev] if base_rev is not None else [] + base_policy_args = ["--base-rev", base_rev] if base_rev is not None else [] + policy_args = _requirement_aware_policy_args(*base_policy_args) _run_hygiene(session, reporter, posargs=["--all-files"], default_all_files=True) _run_policy(session, reporter, *policy_args) _run_lint(session, reporter) diff --git a/tools/policy/historical_identity_records.json b/tools/policy/historical_identity_records.json index 4b2d48b1..78a6e501 100644 --- a/tools/policy/historical_identity_records.json +++ b/tools/policy/historical_identity_records.json @@ -47,7 +47,7 @@ "record_class": "historical-index", "rationale": "Indexes immutable pre-cutover ADR titles, paths, pins, and amendment summaries without making them current identity surfaces.", "occurrences": 4, - "content_sha256": "1112779f7f33a932c90fd5e1bde641199bdf02a9910d89b0207e5ba92ab7562f" + "content_sha256": "fb313cf4bb3281dd64a9c763cf8e0fbc3819e6ea36a6b297045f7fdf45281565" }, { "path": "docs/decisions/adrs/adr-000-use-adrs.md", @@ -495,7 +495,7 @@ "record_class": "historical-index", "rationale": "Indexes immutable pre-cutover ADR titles, paths, pins, and amendment summaries without making them current identity surfaces.", "occurrences": 4, - "content_sha256": "dc023117f930026916eab8c545cf7eaa6ced7245006493c37061032e3f2ecb33" + "content_sha256": "2ec999f5656f32a22acf9f3c94c850fa6328ffd7ebc7e1e06607e5144e8fc05b" }, { "path": "docs/decisions/cage-2-replication-design.md",