diff --git a/docs/MIGRATION_AND_PARITY.md b/docs/MIGRATION_AND_PARITY.md index ab0fdb8..8a51972 100644 --- a/docs/MIGRATION_AND_PARITY.md +++ b/docs/MIGRATION_AND_PARITY.md @@ -74,6 +74,7 @@ Primary Rust gates: cargo fmt --manifest-path rust/Cargo.toml -- --check cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings cargo test --manifest-path rust/Cargo.toml --all-targets +bash scripts/ci/run-quality-indicators.sh ``` Legacy parity harness for retained status-shape compatibility and removed-command diff --git a/docs/QUALITY_GATES.md b/docs/QUALITY_GATES.md new file mode 100644 index 0000000..09eb21f --- /dev/null +++ b/docs/QUALITY_GATES.md @@ -0,0 +1,69 @@ +# Quality gates + +APW keeps PR checks cheap, but release and readiness work should include a +deeper quality signal when code changes affect credential handling, process +execution, or operator diagnostics. + +## Fast checks + +Run the normal local gate before opening a PR: + +```bash +bash scripts/ci/run-fast-checks.sh +cargo fmt --manifest-path rust/Cargo.toml -- --check +cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings +cargo test --manifest-path rust/Cargo.toml +``` + +## Mutation testing + +Mutation testing is opt-in because it is intentionally slower than PR Fast CI. +Install `cargo-mutants`, then run: + +```bash +APW_RUN_MUTATION=1 bash scripts/ci/run-quality-indicators.sh +``` + +The script writes mutation output under `rust/target/mutants` and keeps it out +of source control. When `APW_RUN_MUTATION=1` is set, missing `cargo-mutants` +is a hard failure rather than a silent skip. + +## CRAP indicator + +The CRAP indicator highlights functions whose estimated complexity is high +relative to test coverage: + +```text +complexity^2 * (1 - coverage)^3 + complexity +``` + +Run the source-only hotspot report: + +```bash +bash scripts/ci/run-quality-indicators.sh +``` + +For coverage-aware scores, install `cargo-llvm-cov` and run: + +```bash +APW_RUN_COVERAGE=1 bash scripts/ci/run-quality-indicators.sh +``` + +Without LCOV coverage input, the script assumes 0% coverage and produces a +conservative hotspot list. That mode is useful for deciding where to add tests +before enabling coverage tooling on a runner. When `APW_RUN_COVERAGE=1` is +set, missing `cargo-llvm-cov` is a hard failure rather than a silent skip. +The source scanner recognizes ordinary Rust functions plus qualified forms such +as `async fn`, `const fn`, `unsafe fn`, and `extern "C" fn`, including +visibility-qualified declarations. + +## Tooling regression tests + +Fast checks run: + +```bash +./scripts/test-quality-indicators.sh +``` + +This covers the CRAP self-test and verifies that explicitly requested mutation +or coverage runs fail closed when their required tools are unavailable. diff --git a/docs/SECURITY_POSTURE_AND_TESTING.md b/docs/SECURITY_POSTURE_AND_TESTING.md index f37e273..6a3aeba 100644 --- a/docs/SECURITY_POSTURE_AND_TESTING.md +++ b/docs/SECURITY_POSTURE_AND_TESTING.md @@ -108,8 +108,14 @@ cargo test --manifest-path rust/Cargo.toml --test legacy_parity cargo test --manifest-path rust/Cargo.toml --test native_app_e2e ./scripts/build-universal-release.sh ./scripts/verify-universal-binaries.sh +bash scripts/ci/run-quality-indicators.sh ``` +For high-risk changes, enable the deeper optional checks described in +[QUALITY_GATES.md](QUALITY_GATES.md): `APW_RUN_COVERAGE=1` adds LCOV-backed CRAP +scores when `cargo-llvm-cov` is installed, and `APW_RUN_MUTATION=1` runs +`cargo-mutants` when available. + Before claiming Phase 3 complete for a public release, run the real-hardware notarized broker validation in [PHASE3_HARDWARE_VALIDATION.md](PHASE3_HARDWARE_VALIDATION.md). This check is diff --git a/scripts/ci/run-fast-checks.sh b/scripts/ci/run-fast-checks.sh index 564b845..2329f18 100755 --- a/scripts/ci/run-fast-checks.sh +++ b/scripts/ci/run-fast-checks.sh @@ -44,5 +44,6 @@ done < <(find .github/scripts scripts -type f -name '*.sh' -print0) ./scripts/test-extended-validation-config.sh ./scripts/test-verify-universal-binaries.sh ./scripts/test-universal-release-config.sh +./scripts/test-quality-indicators.sh echo "APW fast checks passed." diff --git a/scripts/ci/run-quality-indicators.sh b/scripts/ci/run-quality-indicators.sh new file mode 100755 index 0000000..dad0c01 --- /dev/null +++ b/scripts/ci/run-quality-indicators.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +echo "Running APW quality indicators..." + +LCOV_PATH="${APW_LCOV_PATH:-}" + +if [[ "${APW_RUN_COVERAGE:-0}" == "1" ]]; then + if command -v cargo-llvm-cov >/dev/null 2>&1; then + LCOV_PATH="${LCOV_PATH:-rust/target/apw-lcov.info}" + cargo llvm-cov \ + --manifest-path rust/Cargo.toml \ + --lcov \ + --output-path "$LCOV_PATH" + else + echo "cargo-llvm-cov is required when APW_RUN_COVERAGE=1." >&2 + echo "Install it or unset APW_RUN_COVERAGE to run the source-only CRAP report." >&2 + exit 1 + fi +fi + +if [[ "${APW_RUN_MUTATION:-0}" == "1" ]]; then + if command -v cargo-mutants >/dev/null 2>&1; then + cargo mutants \ + --manifest-path rust/Cargo.toml \ + --output rust/target/mutants \ + --timeout "${APW_MUTATION_TIMEOUT:-30}" + else + echo "cargo-mutants is required when APW_RUN_MUTATION=1." >&2 + echo "Install it or unset APW_RUN_MUTATION to run CRAP indicators only." >&2 + exit 1 + fi +fi + +CRAP_ARGS=(rust/src --limit "${APW_CRAP_LIMIT:-20}" --threshold "${APW_CRAP_THRESHOLD:-30}") +if [[ -n "$LCOV_PATH" && -f "$LCOV_PATH" ]]; then + CRAP_ARGS+=(--lcov "$LCOV_PATH") +else + echo "No LCOV file supplied; CRAP report uses conservative 0% coverage." >&2 +fi + +python3 scripts/quality/crap_indicator.py "${CRAP_ARGS[@]}" + +echo "APW quality indicators passed." diff --git a/scripts/quality/crap_indicator.py b/scripts/quality/crap_indicator.py new file mode 100755 index 0000000..f729ad8 --- /dev/null +++ b/scripts/quality/crap_indicator.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Compute a lightweight CRAP-style risk indicator for Rust functions. + +The real CRAP score is: + + complexity^2 * (1 - coverage)^3 + complexity + +This script intentionally keeps dependencies at zero. It estimates cyclomatic +complexity from Rust source and optionally reads line coverage from an LCOV +file produced by tools such as `cargo llvm-cov --lcov`. Without LCOV input it +uses 0% coverage, which makes the report a conservative hotspot indicator. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +FN_RE = re.compile( + r'^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:async|const|unsafe)\s+)*(?:extern\s+"[^"]+"\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(' +) +COMPLEXITY_RE = re.compile(r"\b(if|match|for|while)\b|&&|\|\||=>") + + +@dataclass +class FunctionMetric: + path: Path + name: str + start: int + end: int + complexity: int + coverage: float + + @property + def crap(self) -> float: + return (self.complexity**2) * ((1.0 - self.coverage) ** 3) + self.complexity + + +def rust_files(paths: Iterable[Path]) -> list[Path]: + files: list[Path] = [] + for path in paths: + if path.is_file() and path.suffix == ".rs": + files.append(path) + elif path.is_dir(): + files.extend(sorted(p for p in path.rglob("*.rs") if "target" not in p.parts)) + return sorted(files) + + +def line_coverage(lcov: Path | None) -> dict[Path, dict[int, int]]: + if lcov is None: + return {} + coverage: dict[Path, dict[int, int]] = {} + current: Path | None = None + for raw in lcov.read_text(encoding="utf-8").splitlines(): + if raw.startswith("SF:"): + current = Path(raw[3:]).resolve() + coverage.setdefault(current, {}) + elif raw.startswith("DA:") and current is not None: + line_s, hits_s = raw[3:].split(",", 1) + coverage[current][int(line_s)] = int(hits_s) + elif raw == "end_of_record": + current = None + return coverage + + +def function_coverage(path: Path, start: int, end: int, coverage: dict[Path, dict[int, int]]) -> float: + line_hits = coverage.get(path.resolve()) + if not line_hits: + return 0.0 + executable = [line for line in range(start, end + 1) if line in line_hits] + if not executable: + return 0.0 + covered = sum(1 for line in executable if line_hits[line] > 0) + return covered / len(executable) + + +def count_complexity(lines: list[str]) -> int: + complexity = 1 + for line in lines: + stripped = line.split("//", 1)[0] + complexity += len(COMPLEXITY_RE.findall(stripped)) + return complexity + + +def ends_bodyless_function_declaration(line: str, seen_body: bool) -> bool: + return not seen_body and "{" not in line and ";" in line + + +def has_closed_function_body(seen_body: bool, depth: int) -> bool: + return seen_body and depth <= 0 + + +def read_function_body(lines: list[str], start_index: int) -> tuple[list[str], int, bool]: + body_lines: list[str] = [] + depth = 0 + seen_body = False + index = start_index + while index < len(lines): + line = lines[index] + body_lines.append(line) + if ends_bodyless_function_declaration(line, seen_body): + return body_lines, index, False + + depth += line.count("{") + if "{" in line: + seen_body = True + depth -= line.count("}") + if has_closed_function_body(seen_body, depth): + return body_lines, index, True + index += 1 + return body_lines, index - 1, False + + +def parse_functions(path: Path, coverage: dict[Path, dict[int, int]]) -> list[FunctionMetric]: + lines = path.read_text(encoding="utf-8").splitlines() + metrics: list[FunctionMetric] = [] + index = 0 + while index < len(lines): + match = FN_RE.match(lines[index]) + if not match: + index += 1 + continue + name = match.group(1) + start_index = index + body_lines, end_index, has_body = read_function_body(lines, start_index) + if not has_body: + index = end_index + 1 + continue + metrics.append( + FunctionMetric( + path=path, + name=name, + start=start_index + 1, + end=end_index + 1, + complexity=count_complexity(body_lines), + coverage=function_coverage(path, start_index + 1, end_index + 1, coverage), + ) + ) + index = end_index + 1 + return metrics + + +def render_markdown(metrics: list[FunctionMetric], limit: int) -> str: + rows = ["| CRAP | Complexity | Coverage | Function |", "| ---: | ---: | ---: | --- |"] + for metric in metrics[:limit]: + rows.append( + "| " + f"{metric.crap:.1f} | {metric.complexity} | {metric.coverage * 100:.1f}% | " + f"`{metric.path}:{metric.start}` `{metric.name}` |" + ) + return "\n".join(rows) + + +def run(paths: list[Path], lcov: Path | None, limit: int) -> list[FunctionMetric]: + coverage = line_coverage(lcov) + metrics: list[FunctionMetric] = [] + for path in rust_files(paths): + metrics.extend(parse_functions(path, coverage)) + metrics.sort(key=lambda metric: metric.crap, reverse=True) + return metrics[:limit] + + +def self_test() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "sample.rs" + source.write_text( + "\n".join( + [ + "fn simple() {", + " println!(\"ok\");", + "}", + "pub(crate) const fn const_helper() -> bool {", + " true", + "}", + "unsafe fn unsafe_helper(value: bool) {", + " if value { println!(\"unsafe\"); }", + "}", + "pub extern \"C\" fn ffi_helper(value: bool) {", + " if value { println!(\"ffi\"); }", + "}", + "trait Strategy {", + " fn strategy(&self) -> &'static str;", + " fn multiline(", + " &self,", + " ) -> bool;", + " fn generic(", + " &self,", + " value: T,", + " ) -> Result<(), String>;", + "}", + "fn after_trait() {", + " println!(\"after\");", + "}", + "fn branchy(value: bool) {", + " if value && true {", + " match value { true => (), false => () }", + " }", + "}", + ] + ), + encoding="utf-8", + ) + lcov = root / "lcov.info" + lcov.write_text( + "\n".join( + [ + f"SF:{source}", + "DA:1,1", + "DA:2,1", + "DA:3,1", + "DA:4,1", + "DA:5,1", + "DA:6,1", + "DA:7,0", + "DA:8,0", + "DA:9,0", + "DA:10,0", + "DA:11,0", + "DA:12,0", + "DA:13,0", + "DA:14,0", + "DA:15,1", + "DA:16,0", + "DA:17,0", + "DA:18,1", + "end_of_record", + ] + ), + encoding="utf-8", + ) + metrics = run([source], lcov, 10) + names = {metric.name for metric in metrics} + assert {"const_helper", "unsafe_helper", "ffi_helper"}.issubset(names) + assert "strategy" not in names + assert "multiline" not in names + assert "generic" not in names + assert "after_trait" in names + assert metrics[0].name == "branchy" + simple = next(metric for metric in metrics if metric.name == "simple") + assert metrics[0].complexity > simple.complexity + assert simple.coverage == 1.0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*", type=Path, default=[Path("rust/src")]) + parser.add_argument("--lcov", type=Path, help="Optional LCOV file for line coverage.") + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--threshold", type=float, default=30.0) + parser.add_argument("--fail-threshold", action="store_true") + parser.add_argument("--json", action="store_true", dest="json_output") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + self_test() + return 0 + + metrics = run(args.paths, args.lcov, args.limit) + if args.json_output: + print( + json.dumps( + [ + { + "path": str(metric.path), + "line": metric.start, + "name": metric.name, + "complexity": metric.complexity, + "coverage": round(metric.coverage, 4), + "crap": round(metric.crap, 2), + } + for metric in metrics + ], + indent=2, + ) + ) + else: + print(render_markdown(metrics, args.limit)) + + if args.fail_threshold and metrics and metrics[0].crap > args.threshold: + print( + f"CRAP threshold exceeded: {metrics[0].crap:.1f} > {args.threshold:.1f}", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-quality-indicators.sh b/scripts/test-quality-indicators.sh new file mode 100755 index 0000000..83a111e --- /dev/null +++ b/scripts/test-quality-indicators.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +bash -n scripts/ci/run-quality-indicators.sh +python3 scripts/quality/crap_indicator.py --self-test + +EMPTY_PATH="$(mktemp -d "${TMPDIR:-/tmp}/apw-quality-empty-path.XXXXXX")" +trap 'rm -rf "$EMPTY_PATH"' EXIT + +if PATH="$EMPTY_PATH" APW_RUN_MUTATION=1 /bin/bash scripts/ci/run-quality-indicators.sh 2>"$EMPTY_PATH/mutation.err"; then + echo "Expected APW_RUN_MUTATION=1 to fail when cargo-mutants is missing." >&2 + exit 1 +fi +grep -q "cargo-mutants is required when APW_RUN_MUTATION=1" "$EMPTY_PATH/mutation.err" + +if PATH="$EMPTY_PATH" APW_RUN_COVERAGE=1 /bin/bash scripts/ci/run-quality-indicators.sh 2>"$EMPTY_PATH/coverage.err"; then + echo "Expected APW_RUN_COVERAGE=1 to fail when cargo-llvm-cov is missing." >&2 + exit 1 +fi +grep -q "cargo-llvm-cov is required when APW_RUN_COVERAGE=1" "$EMPTY_PATH/coverage.err" + +echo "Quality indicator tests passed."