diff --git a/scripts/ci/check_test_failure_ownership.py b/scripts/ci/check_test_failure_ownership.py new file mode 100644 index 00000000..e764f160 --- /dev/null +++ b/scripts/ci/check_test_failure_ownership.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Test failure ownership attribution report. + +Reads pytest output (from stdin, a file, or a fresh run) and emits a +structured report that groups each failure by its responsible module. + +Usage:: + + # Run pytest internally and report + python scripts/ci/check_test_failure_ownership.py + + # Read from an existing report file + python scripts/ci/check_test_failure_ownership.py --report results.txt + + # Pipe pytest output via stdin + pytest -q --tb=line tests/ | python scripts/ci/check_test_failure_ownership.py --stdin +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +# Ensure repo root is importable so ``from scripts.ci.test_ownership_map`` +# works regardless of how the script is invoked. +_REPO_ROOT = str(Path(__file__).resolve().parents[2]) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +FAILED_PREFIX = "FAILED " +ERROR_PREFIX = "ERROR " +XPASS_PREFIX = "XPASS " +SUMMARY_HEADER = "short test summary info" + + +def _looks_like_test_nodeid(token: str) -> bool: + """Return true when a pytest summary token looks like a real test node id.""" + # ``ERROR path/to/test_file.py`` is still a file-level collection/import failure. + # Only ``.py::...`` tokens identify a concrete pytest test node. + return ".py::" in token + + +def _summary_nodeid(line: str, prefix: str) -> str | None: + """Extract the pytest summary nodeid portion from a FAILED/ERROR line.""" + if not line.startswith(prefix): + return None + remainder = line[len(prefix) :] + if not remainder: + return None + + bracket_depth = 0 + for idx, char in enumerate(remainder): + if char == "[": + bracket_depth += 1 + continue + if char == "]" and bracket_depth > 0: + bracket_depth -= 1 + continue + if bracket_depth == 0 and remainder.startswith(" - ", idx): + return remainder[:idx] + return remainder + + +def _candidate_summary_lines(text: str) -> list[str]: + """Prefer the pytest short-summary section when the full report is available.""" + lines = text.splitlines() + for index, line in enumerate(lines): + if SUMMARY_HEADER in line.lower(): + return lines[index + 1 :] + return lines + + +def _parse_failed_lines(text: str) -> list[str]: + """Extract pytest failure-like summary node IDs from raw pytest output.""" + results: list[str] = [] + for line in _candidate_summary_lines(text): + stripped = line.strip() + for prefix in (FAILED_PREFIX, ERROR_PREFIX, XPASS_PREFIX): + nodeid = _summary_nodeid(stripped, prefix) + if nodeid is not None and _looks_like_test_nodeid(nodeid): + results.append(nodeid) + break + return results + + +def _has_unattributed_pytest_error(text: str) -> bool: + """Detect pytest process failures that did not yield attributable node IDs.""" + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + normalized = stripped.lower() + if stripped.startswith(("ERROR:", "ERROR collecting ", "INTERNALERROR>")): + return True + if "KeyboardInterrupt" in stripped or "NO_TESTS_COLLECTED" in stripped: + return True + if normalized.startswith("no tests ran") or "no tests collected" in normalized: + return True + error_nodeid = _summary_nodeid(stripped, ERROR_PREFIX) + if error_nodeid is not None and not _looks_like_test_nodeid(error_nodeid): + return True + return False + + +def _test_path_from_nodeid(nodeid: str) -> str: + """Convert ``tests/unit/test_foo.py::test_bar`` to ``tests/unit/test_foo.py``.""" + return nodeid.split("::")[0] + + +def _build_report(failed_nodeids: list[str]) -> str: + from scripts.ci.test_ownership_map import OWNERSHIP_MAP + + if not failed_nodeids: + return "=== Test Failure Ownership Report ===\n\nNo test failures detected.\n" + + # Group by module + module_groups: dict[str, list[str]] = {} + unmapped: list[str] = [] + + for nodeid in failed_nodeids: + path = _test_path_from_nodeid(nodeid) + entry = OWNERSHIP_MAP.get(path) + if entry: + module_groups.setdefault(entry["module"], []).append(nodeid) + else: + unmapped.append(nodeid) + + lines: list[str] = ["=== Test Failure Ownership Report ===", ""] + + for module in sorted(module_groups): + # Find owner from first entry in this module group + first_path = _test_path_from_nodeid(module_groups[module][0]) + entry = OWNERSHIP_MAP.get(first_path, {}) + owner = entry.get("owner") or "unassigned" + lines.append(f"Module: {module} (owner: {owner})") + for nodeid in sorted(module_groups[module]): + lines.append(f" FAILED {nodeid}") + lines.append("") + + lines.append("UNMAPPED (no ownership entry):") + if unmapped: + for nodeid in sorted(unmapped): + lines.append(f" FAILED {nodeid}") + else: + lines.append(" (none)") + lines.append("") + + total_failed = len(failed_nodeids) + total_modules = len(module_groups) + (1 if unmapped else 0) + lines.append(f"Summary: {total_failed} failed tests across {total_modules} modules") + return "\n".join(lines) + + +def _format_completed_output(completed: subprocess.CompletedProcess[str]) -> str: + """Join stdout/stderr into the single text blob consumed by the parser.""" + return "\n".join( + part for part in [completed.stdout.strip(), completed.stderr.strip()] if part + ).strip() + + +def _run_pytest() -> subprocess.CompletedProcess[str]: + """Run ``pytest -q --tb=line tests/`` and return the completed process.""" + return subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--tb=line", "tests/"], + capture_output=True, + text=True, + check=False, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Test failure ownership attribution report") + source = parser.add_mutually_exclusive_group() + source.add_argument( + "--stdin", + action="store_true", + help="Read pytest output from stdin", + ) + source.add_argument( + "--report", + type=Path, + metavar="FILE", + help="Read pytest output from a file", + ) + args = parser.parse_args(argv) + + if args.stdin: + raw = sys.stdin.read() + pytest_returncode: int | None = None + if not raw.strip(): + # The documented pipe flow should yield pytest stdout. Empty stdin + # means the upstream command failed before producing reportable output. + print("No pytest output received on stdin.", file=sys.stderr) + return 1 + elif args.report: + raw = args.report.read_text(encoding="utf-8") + pytest_returncode = None + if not raw.strip(): + print("No pytest output found in report file.", file=sys.stderr) + return 1 + else: + completed = _run_pytest() + raw = _format_completed_output(completed) + pytest_returncode = completed.returncode + + failed = _parse_failed_lines(raw) + has_unattributed_error = _has_unattributed_pytest_error(raw) + fallback_exit_code = pytest_returncode + if fallback_exit_code is None and has_unattributed_error: + fallback_exit_code = 1 + if has_unattributed_error and raw: + print(raw, file=sys.stderr) + if not failed and fallback_exit_code: + return fallback_exit_code + + report = _build_report(failed) + print(report) + + if failed: + return fallback_exit_code or 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/test_ownership_map.py b/scripts/ci/test_ownership_map.py new file mode 100644 index 00000000..de37625a --- /dev/null +++ b/scripts/ci/test_ownership_map.py @@ -0,0 +1,637 @@ +"""Test file -> responsible module ownership mapping. + +Canonical source for test failure attribution. Used by: +- tests/conftest.py (pytest marker injection) +- scripts/ci/check_test_failure_ownership.py (failure attribution report) +""" + +from __future__ import annotations + +from pathlib import Path, PurePath + + +def is_pytest_discovery_file(path: str | Path | PurePath) -> bool: + """Match pytest's default module filename patterns used by this repo.""" + name = PurePath(path).name + return name.startswith("test_") or name.endswith("_test.py") + +# Each entry: test_path (relative to repo root) -> { module, owner, tier } +# owner: responsible individual or team handle +OWNERSHIP_MAP: dict[str, dict[str, str]] = { + # --- integration tests --- + "tests/integration/test_client_cli_flow.py": { + "module": "client", + "owner": "lang", + "tier": "integration", + }, + "tests/integration/test_config_provider_reload.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "integration", + }, + "tests/integration/test_example_agent_flow.py": { + "module": "examples", + "owner": "lang", + "tier": "integration", + }, + "tests/integration/test_hook_governance_flow.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "integration", + }, + "tests/integration/test_p0_conformance_gate.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "integration", + }, + "tests/integration/test_security_policy_gate_flow.py": { + "module": "dare_framework/security", + "owner": "lang", + "tier": "integration", + }, + # --- smoke tests --- + "tests/smoke/test_ci_smoke.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "smoke", + }, + "tests/smoke/test_client_cli_smoke.py": { + "module": "client", + "owner": "lang", + "tier": "smoke", + }, + "tests/smoke/test_client_entrypoint_smoke.py": { + "module": "client", + "owner": "lang", + "tier": "smoke", + }, + # --- unit tests: a2a --- + "tests/unit/test_a2a.py": { + "module": "dare_framework/a2a", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: agent domain --- + "tests/unit/test_agent_construction_contract.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_agent_event_transport_hook.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_agent_input_normalizer.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_agent_output_envelope.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_base_agent_transport_contract.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_component_managers.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_config_filtering.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_hook_ordering.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_manager_resolution.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_security_boundary.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_skill_tool_mode.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builder_tool_gateway.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_component_managers.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_hook_governance.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_hook_transport_boundary.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_mcp_management.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_orchestration_split.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_security_boundary.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_security_policy_gate.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_dare_agent_step_driven_mode.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_five_layer_agent.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_kernel_flow.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_output_normalizer.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_react_agent_gateway_injection.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_runtime_state.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_session_orchestrator_message_input.py": { + "module": "dare_framework/agent", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: checkpoint --- + "tests/unit/test_checkpoint_message_schema.py": { + "module": "dare_framework/checkpoint", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: client --- + "tests/unit/test_client_cli.py": { + "module": "client", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_client_runtime_bootstrap.py": { + "module": "client", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: config --- + "tests/unit/test_config_action_handler.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_config_model.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_config_nonnull_guards.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_config_provider.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_layered_config_provider.py": { + "module": "dare_framework/config", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: context --- + "tests/unit/test_context_implementation.py": { + "module": "dare_framework/context", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_context_message_types.py": { + "module": "dare_framework/context", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: event --- + "tests/unit/test_event_log.py": { + "module": "dare_framework/event", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_event_sqlite_event_log.py": { + "module": "dare_framework/event", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: examples --- + "tests/unit/test_example_10_agentscope_compat.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_example_hook_governance.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_examples_04_cli.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_examples_cli.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_examples_cli_mcp.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_examples_with_tools.py": { + "module": "examples", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: hook --- + "tests/unit/test_hook_config_model.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_contract_types.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_decision_arbiter.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_extension_point_governance.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_legacy_adapter.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_metrics.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_patch_validator.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_phase_schema.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_runner.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_hook_selector.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_stdout_hook.py": { + "module": "dare_framework/hook", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: governance / CI scripts --- + "tests/unit/test_governance_evidence_truth_gate.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_governance_intent_gate.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_governance_traceability_gate.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_check_test_failure_ownership.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_conftest_ownership_markers.py": { + "module": "tests", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_main_guard.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_manual_merge_guard.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_p0_gate_ci.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: interaction / transport --- + "tests/unit/test_interaction_controls.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_interaction_dispatcher.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_transport_adapters.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_transport_channel.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_transport_typed_payloads.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_transport_types.py": { + "module": "dare_framework/transport", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: mcp --- + "tests/unit/test_mcp_action_handler.py": { + "module": "dare_framework/mcp", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_mcp_client.py": { + "module": "dare_framework/mcp", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_mcp_manager.py": { + "module": "dare_framework/mcp", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_mcp_tool_provider.py": { + "module": "dare_framework/mcp", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: memory --- + "tests/unit/test_memory_knowledge_direct.py": { + "module": "dare_framework/memory", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_embedding_openai_adapter.py": { + "module": "dare_framework/memory", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: model --- + "tests/unit/test_anthropic_model_adapter.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_builtin_prompt_loader.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_default_model_adapter_manager.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_openai_model_adapter.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_openrouter_adapter.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_prompt_store.py": { + "module": "dare_framework/model", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: observability --- + "tests/unit/test_llm_io_capture_hook.py": { + "module": "dare_framework/observability", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_observability.py": { + "module": "dare_framework/observability", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: plan --- + "tests/unit/test_composite_validator.py": { + "module": "dare_framework/plan", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_default_planner.py": { + "module": "dare_framework/plan", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_default_remediator.py": { + "module": "dare_framework/plan", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_plan_v2_tools.py": { + "module": "dare_framework/plan", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_registry_plan_validator.py": { + "module": "dare_framework/plan", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: security --- + "tests/unit/test_security_boundary.py": { + "module": "dare_framework/security", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: skill --- + "tests/unit/test_search_skill_tool.py": { + "module": "dare_framework/skill", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_skill_store_builder.py": { + "module": "dare_framework/skill", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: tool --- + "tests/unit/test_execution_control.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_governed_tool_gateway.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_native_tool_provider.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_approval_action_handler.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_approval_manager.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_gateway.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_manager.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_signature_contract.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_tool_types.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_v4_file_tools.py": { + "module": "dare_framework/tool", + "owner": "lang", + "tier": "unit", + }, + # --- unit tests: cross-cutting --- + "tests/unit/test_llm_io_summary_script.py": { + "module": "scripts", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_local_backend.py": { + "module": "_local_backend", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_package_initializers_facade_pattern.py": { + "module": "dare_framework", + "owner": "lang", + "tier": "unit", + }, + "tests/unit/test_ownership_map_coverage.py": { + "module": "scripts/ci", + "owner": "lang", + "tier": "unit", + }, +} + + +def get_entry(test_path: str) -> dict[str, str] | None: + """Return the ownership entry for *test_path* (relative to repo root).""" + return OWNERSHIP_MAP.get(test_path) + + +def modules_for_failures(failed_paths: list[str]) -> dict[str, list[str]]: + """Group *failed_paths* by their responsible module. + + Returns ``{module: [test_path, ...]}``. Unmapped paths are grouped + under the key ``"UNMAPPED"``. + """ + grouped: dict[str, list[str]] = {} + for path in failed_paths: + entry = OWNERSHIP_MAP.get(path) + key = entry["module"] if entry else "UNMAPPED" + grouped.setdefault(key, []).append(path) + return grouped diff --git a/tests/conftest.py b/tests/conftest.py index 2a855d9f..88defd07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,50 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) + + +# --------------------------------------------------------------------------- +# Marker registration & automatic injection from canonical ownership map +# --------------------------------------------------------------------------- + + +def pytest_configure(config: pytest.Config) -> None: + """Register ``module`` and ``owner`` markers so they pass ``--strict-markers``.""" + config.addinivalue_line("markers", "module(name): responsible dare_framework module") + config.addinivalue_line("markers", "owner(name): responsible owner/team handle") + + +def _normalize_ownership_relpath(path: Path) -> str: + """Normalize repo-relative test paths into OWNERSHIP_MAP key format.""" + return path.as_posix() + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Inject ``@pytest.mark.module`` / ``@pytest.mark.owner`` from the ownership map.""" + import warnings + + from scripts.ci.test_ownership_map import OWNERSHIP_MAP, is_pytest_discovery_file + + for item in items: + rel = _normalize_ownership_relpath(Path(item.fspath).relative_to(ROOT)) + entry = OWNERSHIP_MAP.get(rel) + if entry: + item.add_marker(pytest.mark.module(entry["module"])) + if entry.get("owner"): + item.add_marker(pytest.mark.owner(entry["owner"])) + + # Warn about test files missing from OWNERSHIP_MAP (soft guard) + collected_files = {_normalize_ownership_relpath(Path(item.fspath).relative_to(ROOT)) for item in items} + test_files = {f for f in collected_files if is_pytest_discovery_file(f)} + unmapped = test_files - set(OWNERSHIP_MAP.keys()) + if unmapped: + warnings.warn( + f"Test files missing from OWNERSHIP_MAP ({len(unmapped)}): " + + ", ".join(sorted(unmapped)[:5]), + stacklevel=1, + ) diff --git a/tests/unit/test_check_test_failure_ownership.py b/tests/unit/test_check_test_failure_ownership.py new file mode 100644 index 00000000..d86fef18 --- /dev/null +++ b/tests/unit/test_check_test_failure_ownership.py @@ -0,0 +1,269 @@ +import io +import subprocess + +from scripts.ci import check_test_failure_ownership as module + + +def test_main_propagates_pytest_process_errors_without_failed_nodeids( + monkeypatch, capsys +) -> None: + def _fake_run(*_args, **_kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["pytest"], + returncode=4, + stdout="", + stderr="ERROR: usage: pytest [options] [file_or_dir] ...", + ) + + monkeypatch.setattr(module.subprocess, "run", _fake_run) + + exit_code = module.main([]) + + captured = capsys.readouterr() + assert exit_code == 4 + assert "ERROR: usage: pytest" in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_propagates_pytest_collection_errors_without_fake_failed_nodeid( + monkeypatch, capsys +) -> None: + def _fake_run(*_args, **_kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["pytest"], + returncode=2, + stdout="ERROR collecting tests/unit/test_demo.py", + stderr="", + ) + + monkeypatch.setattr(module.subprocess, "run", _fake_run) + + exit_code = module.main([]) + + captured = capsys.readouterr() + assert exit_code == 2 + assert "ERROR collecting tests/unit/test_demo.py" in captured.err + assert "FAILED collecting" not in captured.out + + +def test_main_propagates_pytest_file_level_errors_without_pseudo_nodeid( + monkeypatch, capsys +) -> None: + def _fake_run(*_args, **_kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["pytest"], + returncode=2, + stdout="ERROR tests/unit/test_tmp_collection_error.py", + stderr="", + ) + + monkeypatch.setattr(module.subprocess, "run", _fake_run) + + exit_code = module.main([]) + + captured = capsys.readouterr() + assert exit_code == 2 + assert "ERROR tests/unit/test_tmp_collection_error.py" in captured.err + assert "FAILED tests/unit/test_tmp_collection_error.py" not in captured.out + + +def test_main_preserves_unattributed_pytest_errors_alongside_failed_nodeids( + monkeypatch, capsys +) -> None: + def _fake_run(*_args, **_kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["pytest"], + returncode=2, + stdout="\n".join( + [ + "FAILED tests/unit/test_check_test_failure_ownership.py::test_parse_failed_lines_preserves_parametrized_nodeids_with_spaces - AssertionError: boom", + "ERROR collecting tests/unit/test_tmp_collection_error.py", + ] + ), + stderr="", + ) + + monkeypatch.setattr(module.subprocess, "run", _fake_run) + + exit_code = module.main([]) + + captured = capsys.readouterr() + assert exit_code == 2 + assert "ERROR collecting tests/unit/test_tmp_collection_error.py" in captured.err + assert ( + "FAILED tests/unit/test_check_test_failure_ownership.py::" + "test_parse_failed_lines_preserves_parametrized_nodeids_with_spaces" + ) in captured.out + + +def test_main_stdin_mode_returns_nonzero_for_collection_errors_without_failed_nodeids( + monkeypatch, capsys +) -> None: + monkeypatch.setattr( + module.sys, + "stdin", + io.StringIO("ERROR collecting tests/unit/test_demo.py"), + ) + + exit_code = module.main(["--stdin"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "ERROR collecting tests/unit/test_demo.py" in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_report_mode_returns_nonzero_for_usage_errors_without_failed_nodeids( + tmp_path, capsys +) -> None: + report_path = tmp_path / "pytest-output.txt" + report_path.write_text( + "ERROR: usage: pytest [options] [file_or_dir] ...", + encoding="utf-8", + ) + + exit_code = module.main(["--report", str(report_path)]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "ERROR: usage: pytest" in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_report_mode_returns_nonzero_for_empty_report_file( + tmp_path, capsys +) -> None: + report_path = tmp_path / "pytest-output.txt" + report_path.write_text("", encoding="utf-8") + + exit_code = module.main(["--report", str(report_path)]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "No pytest output found in report file." in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_stdin_mode_returns_nonzero_for_keyboard_interrupt( + monkeypatch, capsys +) -> None: + monkeypatch.setattr( + module.sys, + "stdin", + io.StringIO("KeyboardInterrupt"), + ) + + exit_code = module.main(["--stdin"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "KeyboardInterrupt" in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_stdin_mode_returns_nonzero_for_empty_input(monkeypatch, capsys) -> None: + monkeypatch.setattr(module.sys, "stdin", io.StringIO("")) + + exit_code = module.main(["--stdin"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "No pytest output received on stdin." in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_report_mode_returns_nonzero_when_no_tests_are_collected( + tmp_path, capsys +) -> None: + report_path = tmp_path / "pytest-output.txt" + report_path.write_text( + "no tests ran in 0.01s", + encoding="utf-8", + ) + + exit_code = module.main(["--report", str(report_path)]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "no tests ran in 0.01s" in captured.err + assert "No test failures detected." not in captured.out + + +def test_main_report_mode_returns_nonzero_for_strict_xpass_summary( + tmp_path, capsys +) -> None: + report_path = tmp_path / "pytest-output.txt" + report_path.write_text( + "\n".join( + [ + "=========================== short test summary info ============================", + "XPASS tests/unit/test_check_test_failure_ownership.py::test_parse_failed_lines_ignores_non_node_failed_output - unexpectedly passed", + ] + ), + encoding="utf-8", + ) + + exit_code = module.main(["--report", str(report_path)]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert ( + "FAILED tests/unit/test_check_test_failure_ownership.py::" + "test_parse_failed_lines_ignores_non_node_failed_output" + ) in captured.out + assert "No test failures detected." not in captured.out + + +def test_parse_failed_lines_preserves_parametrized_nodeids_with_spaces() -> None: + raw = "\n".join( + [ + "FAILED tests/unit/test_demo.py::test_case[hello world] - AssertionError: boom", + "ERROR tests/unit/test_demo.py::test_setup[hello world] - RuntimeError: boom", + ] + ) + + assert module._parse_failed_lines(raw) == [ + "tests/unit/test_demo.py::test_case[hello world]", + "tests/unit/test_demo.py::test_setup[hello world]", + ] + + +def test_parse_failed_lines_preserves_nodeids_containing_dash_delimiters() -> None: + raw = "\n".join( + [ + "FAILED tests/unit/test_demo.py::test_case[hello - world] - AssertionError: boom", + "ERROR tests/unit/test_demo.py::test_setup[hello - world] - RuntimeError: boom", + ] + ) + + assert module._parse_failed_lines(raw) == [ + "tests/unit/test_demo.py::test_case[hello - world]", + "tests/unit/test_demo.py::test_setup[hello - world]", + ] + + +def test_parse_failed_lines_ignores_non_node_failed_output() -> None: + raw = "\n".join( + [ + "FAILED not-a-nodeid", + "FAILED tests/unit/test_demo.py::test_case - AssertionError: boom", + ] + ) + + assert module._parse_failed_lines(raw) == [ + "tests/unit/test_demo.py::test_case", + ] + + +def test_parse_failed_lines_ignores_failed_like_log_lines_before_summary() -> None: + raw = "\n".join( + [ + "FAILED tests/unit/test_other.py::test_noise - user log line", + "=========================== short test summary info ============================", + "FAILED tests/unit/test_demo.py::test_case - AssertionError: boom", + ] + ) + + assert module._parse_failed_lines(raw) == [ + "tests/unit/test_demo.py::test_case", + ] diff --git a/tests/unit/test_conftest_ownership_markers.py b/tests/unit/test_conftest_ownership_markers.py new file mode 100644 index 00000000..ee582312 --- /dev/null +++ b/tests/unit/test_conftest_ownership_markers.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path, PureWindowsPath + +from scripts.ci.test_ownership_map import is_pytest_discovery_file + + +def _load_repo_conftest(): + repo_root = Path(__file__).resolve().parents[2] + spec = importlib.util.spec_from_file_location( + "repo_test_conftest", + repo_root / "tests" / "conftest.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_normalize_ownership_relpath_uses_forward_slashes() -> None: + repo_conftest = _load_repo_conftest() + + assert repo_conftest._normalize_ownership_relpath( # type: ignore[attr-defined] + PureWindowsPath("tests\\unit\\test_demo.py") + ) == "tests/unit/test_demo.py" + + +def test_is_pytest_discovery_file_matches_default_filename_patterns() -> None: + assert is_pytest_discovery_file("tests/unit/test_demo.py") + assert is_pytest_discovery_file("tests/unit/demo_test.py") + assert not is_pytest_discovery_file("tests/unit/demo.py") diff --git a/tests/unit/test_ownership_map_coverage.py b/tests/unit/test_ownership_map_coverage.py new file mode 100644 index 00000000..9a06eb4d --- /dev/null +++ b/tests/unit/test_ownership_map_coverage.py @@ -0,0 +1,35 @@ +"""Verify OWNERSHIP_MAP stays in sync with actual test files.""" +from pathlib import Path + +from scripts.ci.test_ownership_map import OWNERSHIP_MAP, is_pytest_discovery_file + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def test_all_test_files_have_ownership_entry(): + """Every pytest-discoverable test module must be present in OWNERSHIP_MAP.""" + actual = sorted( + p.relative_to(REPO_ROOT).as_posix() + for p in REPO_ROOT.joinpath("tests").rglob("*.py") + if is_pytest_discovery_file(p) + ) + mapped = set(OWNERSHIP_MAP.keys()) + missing = [f for f in actual if f not in mapped] + assert not missing, f"Test files missing from OWNERSHIP_MAP: {missing}" + + +def test_no_stale_ownership_entries(): + """Every OWNERSHIP_MAP entry must correspond to an existing test file.""" + actual = set( + p.relative_to(REPO_ROOT).as_posix() + for p in REPO_ROOT.joinpath("tests").rglob("*.py") + if is_pytest_discovery_file(p) + ) + stale = sorted(k for k in OWNERSHIP_MAP if k not in actual) + assert not stale, f"Stale OWNERSHIP_MAP entries (files no longer exist): {stale}" + + +def test_all_owners_are_nonempty(): + """Every entry must have a non-empty owner assigned.""" + unassigned = sorted(k for k, v in OWNERSHIP_MAP.items() if not v.get("owner")) + assert not unassigned, f"Entries with no owner assigned: {unassigned[:10]}..."