diff --git a/.github/workflows/parity-tier2.yml b/.github/workflows/parity-tier2.yml new file mode 100644 index 0000000..1bf39f4 --- /dev/null +++ b/.github/workflows/parity-tier2.yml @@ -0,0 +1,103 @@ +# Feature 028 Tier 2: coding-agent skill vs raw MCP tool output parity. +# +# Manual-dispatch only. See contract at +# specs/028-audit-parity-tests/contracts/tier2-workflow.md +# and governance rationale at spec.md FR-007 / FR-007a / FR-007b. +# +# Key security properties (see contract T2-1..T2-16): +# - T2-1: workflow_dispatch is the ONLY trigger. No schedule; no push. +# - T2-2: Environment `parity-tier2` is REQUIRED. Configure in GitHub UI +# with a reviewer list AND the ANTHROPIC_API_KEY secret at the +# Environment level (NOT repo level). +# - T2-4: NO other workflow in .github/workflows/ references +# secrets.ANTHROPIC_API_KEY. Verified by +# tests/darnit/parity/tier2/test_workflow_config.py. +# - T2-5: permissions: contents: read only. No write scope granted. +# - T2-7/T2-8: Preflight actor+SHA logged BEFORE the SDK step consumes +# the API key. +# - T2-10: NO api_key workflow input (governance regression guard). +# - T2-11: artifact upload runs on any exit code (`if: always()`). +# +# Kusari Inspector hardening (2026-08-12): +# - Actions pinned to full commit SHAs (mutable version tags would allow +# a compromised action owner to silently repoint a tag and execute +# arbitrary code in the runner while ANTHROPIC_API_KEY is in env). +# - All `${{ ... }}` context expressions in `run:` blocks moved to +# step-level `env:` variables and referenced as quoted shell vars. +# Prevents shell injection via crafted `fixture_glob` inputs. + +name: Parity Tier 2 + +on: + workflow_dispatch: + inputs: + fixture_glob: + description: "Glob to filter which fixtures are run (default: all)" + default: "*" + required: false + +permissions: + contents: read + +jobs: + tier2: + runs-on: ubuntu-latest + environment: parity-tier2 # T2-2: gated Environment with required reviewers + permissions: + contents: read # T2-5: no write scope + + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + + - name: Sync Tier 2 parity environment + # PR #370 review fix: `claude-agent-sdk` no longer ships in the + # `[dev]` extra of `darnit-mcp`; it lives in the dedicated + # `[parity-tier2]` extra so `pip install darnit-mcp[dev]` on a + # regular contributor's machine stays lean. Tier 2 CI opts in + # explicitly. + run: uv sync --extra dev --extra parity-tier2 + + - name: Preflight audit log (T2-7/T2-8) + env: + ACTOR: ${{ github.actor }} + SHA: ${{ github.sha }} + FIXTURE_GLOB: ${{ inputs.fixture_glob }} + run: | + { + echo "## Tier 2 Preflight" + echo "" + echo "- actor: $ACTOR" + echo "- sha: $SHA" + echo "- fixture_glob: $FIXTURE_GLOB" + echo "- timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run Tier 2 parity check + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + FIXTURE_GLOB: ${{ inputs.fixture_glob }} + run: | + # Run as a module (`python -m`) so the `tests.` package imports + # inside run.py resolve. Running as a script (`python `) + # would fail because the `tests` package wouldn't be on sys.path. + # FIXTURE_GLOB is passed via env (not interpolated directly into + # this shell block) to prevent shell injection. + uv run python -m tests.darnit.parity.tier2.run \ + --fixture-glob "$FIXTURE_GLOB" + + - name: Upload parity artifacts (T2-11) + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: parity-artifacts + path: parity-artifacts/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index 3db0968..0479bf6 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,11 @@ project.toml !example.*.toml !example.*.yaml # Test fixtures with tracked .baseline.toml. The top-level rule above -# would otherwise strip them and CI would auto-pick a different -# framework than the fixture expects (harness/parity tests). +# would otherwise strip them and CI would either deselect the whole +# suite (parity) or auto-pick a different framework than the fixture +# expects (harness). !tests/darnit/harness/fixtures/**/.baseline.toml +!tests/darnit/parity/fixtures/**/.baseline.toml # Logs *.log @@ -51,3 +53,6 @@ baseline-test-repo/ .doc-cache/ .darnit/ + +# Feature 028: Tier 2 parity check writes skill-invocation transcripts here. +parity-artifacts/ diff --git a/.specify/feature.json b/.specify/feature.json index 0963034..eb972db 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1 +1 @@ -{"feature_directory": "specs/027-interactive-resolvers"} +{"feature_directory": "specs/028-audit-parity-tests"} diff --git a/CLAUDE.md b/CLAUDE.md index d930060..7137952 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -369,6 +369,7 @@ else: - Filesystem only. Composition is resolved in-memory at framework-config load time; no new persistent state. (013-plugin-composition) ## Recent Changes +- 028-audit-parity-tests: two-tier parity test suite verifying the darnit audit's per-control output is consistent across consumers. Tier 1 (`tests/darnit/parity/tier1/`) runs on every PR: parametrized-per-fixture pytest that invokes both the direct `audit_openssf_baseline` MCP tool AND `HarnessRun` (with `MockLLMStep`) in-process, then diffs per-control status. Sole allowed drift is PENDING_LLM (MCP) -> non-PENDING_LLM (harness). Tier 2 (`tests/darnit/parity/tier2/`) is manual-dispatch only via `.github/workflows/parity-tier2.yml` -- Environment-gated with required reviewers, no repo-level `ANTHROPIC_API_KEY` exposure. Uses `claude-agent-sdk` (test-only dev dep) to invoke the `/darnit-audit` skill, parses its final assistant message, diffs against the raw MCP tool JSON. Fixture corpus at `tests/darnit/parity/fixtures/`; `parity.toml` per fixture declares expected shape + `control_ids` filter. Zero product-package changes (SC-006), enforced by a git-diff-based test. Closes #366. Follow-up issues: #368 (OpenAI SDK parity), #369 (scheduled cadence + governance-appropriate key sourcing). - 027-interactive-resolvers: adds `--interactive` flag to `darnit harness` and a new `QuestionResolver` Protocol (async, `@runtime_checkable`) that sits downstream of feature 026's `AnswerSource` chain. `InteractiveTerminalResolver` reference implementation prompts on `/dev/tty` (isolated from stdout report / stderr progress streams). Third-party resolvers register via Python entry points under group `darnit.question_resolvers` (mirrors `darnit.frameworks` discovery). Every `Answer` carries `authority: "asserted"` enforced at the model layer via `Literal["asserted"]` with a fixed default. Per-question `resolution_trail` in the report captures which resolvers were offered a question and how each responded (`answered`/`skipped`/`errored`). Fail-fast (<2s) when stdin is not a TTY OR /dev/tty is not openable under `--interactive`. Feature 026's "no re-audit after collect" MVP policy preserved. - 026-darnit-harness: adds `darnit harness` subcommand -- end-to-end audit driver with in-band LLM dispatch (fleet-operator + CI-integrated persona). Consumes `ANTHROPIC_API_KEY` from env; dispatches PENDING_LLM results via `PydanticAILLMStep`. Non-interactive by default; batch answers via pluggable `AnswerSource` Protocol with auto-discovery of `.project/project.yaml` + `--answers` override. Markdown + JSON reports. Four documented exit codes (0/1/2/3) plus grep-able stderr summary. New `darnit.harness` subpackage (`driver`, `answer_sources`, `report`, `exit_codes`). - 025-rfc0001-stage1: RFC-0001 Stage 1. Adds `authority` (`dispositive`|`suggestive`|`asserted`) to every step + result; per-phase Check execution rule ensures only dispositive/asserted results conclude a control (LLM output alone cannot manufacture a PASS). New `darnit.core.action_plan` module exposes `next_action`/`submit_result` as a public typed protocol; `agent.graph.route()` becomes a thin adapter. MCP surface adds `run_next_action`/`submit_action_result` tools (client-owned state). Baseline attestation predicate gains a per-result `authority` field additively within v1. `pydantic-ai-slim[anthropic]` becomes a required runtime dep. @@ -379,5 +380,5 @@ else: For additional context about technologies to be used, project structure, shell commands, and other important information, read the current plan: -[`specs/027-interactive-resolvers/plan.md`](specs/027-interactive-resolvers/plan.md) +[`specs/028-audit-parity-tests/plan.md`](specs/028-audit-parity-tests/plan.md) diff --git a/pyproject.toml b/pyproject.toml index b342e2b..76e9b11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,13 @@ dev = [ "pre-commit>=4.0.0", "vulture>=2.11", ] +# Feature 028 Tier 2 (PR #370): the coding-agent parity check imports +# `claude_agent_sdk` (~90 MB). Isolate it here so `darnit-mcp[dev]` +# stays lean -- Tier 2 CI installs `darnit-mcp[parity-tier2]` +# explicitly. Not part of `[dev]` per PR #370 review feedback. +parity-tier2 = [ + "claude-agent-sdk>=0.1.0", +] [tool.uv.workspace] members = ["packages/*"] diff --git a/specs/028-audit-parity-tests/checklists/requirements.md b/specs/028-audit-parity-tests/checklists/requirements.md new file mode 100644 index 0000000..2e95353 --- /dev/null +++ b/specs/028-audit-parity-tests/checklists/requirements.md @@ -0,0 +1,73 @@ +# Specification Quality Checklist: Two-Tier Audit Parity Tests + +**Purpose**: Validate specification completeness and quality before proceeding to planning + +**Created**: 2026-08-09 + +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [X] No implementation details (languages, frameworks, APIs) +- [X] Focused on user value and business needs +- [X] Written for non-technical stakeholders +- [X] All mandatory sections completed + +## Requirement Completeness + +- [X] No [NEEDS CLARIFICATION] markers remain +- [X] Requirements are testable and unambiguous +- [X] Success criteria are measurable +- [X] Success criteria are technology-agnostic (no implementation details) +- [X] All acceptance scenarios are defined +- [X] Edge cases are identified +- [X] Scope is clearly bounded +- [X] Dependencies and assumptions identified + +## Feature Readiness + +- [X] All functional requirements have clear acceptance criteria +- [X] User scenarios cover primary flows +- [X] Feature meets measurable outcomes defined in Success Criteria +- [X] No implementation details leak into specification + +## Notes + +- This spec is diagnostic, not remedial: it defines a test surface that WILL detect drift between the three audit consumers. It does NOT define fixes for any specific drift the tests might discover. That intentional scoping keeps the feature small and its purpose clear. +- Constitution IV echo: the whole reason this feature exists is that a downstream layer (the `/darnit-audit` coding-agent skill) was observed silently reinterpreting the tool's verdicts. Tier 2 makes that behavior visible; a fix for it would be a separate feature. +- Feature dependencies: 026 (harness) is hard-required for Tier 1. 027 (interactive resolvers) is intentionally out of scope; parity tests do not exercise interactive answer collection. +- Two areas were considered for [NEEDS CLARIFICATION] but resolved with defaults instead: + - Tier 2 cadence: chose "nightly or weekly, plan-phase decides." A specific number would over-fit the spec. + - Claude Agent SDK vs Claude Code CLI subprocess: chose "SDK if available, CLI subprocess as fallback." The plan phase pins the exact choice. +- The spec commits to closing issue #366 on merge (FR-016 + SC-009). This is the audit trail linking the surfaced problem to the shipped diagnostic. + +## Clarification Session Log + +Five clarifications recorded during the 2026-08-09 clarify session: + +1. **Tier 2 invocation mechanism** -> Claude Agent SDK (test-only dep). Follow-up issue #368 opened for OpenAI-SDK and other-provider parity checks; those are separate features. +2. **Skill's summary artifact** -> Final assistant message. Diagnostic feature must compare user-facing output; structured-artifact alternatives rejected as intrusive. +3. **Tier 2 cadence** -> Manual-only for MVP (`workflow_dispatch`); no schedule. Governance driver: repo is under neutral governance, API key belongs to a specific company. FR-007a + FR-007b + SC-005a lock down the access-control shape. Follow-up issue #369 opened for adding scheduled cadence + governance-appropriate key-sourcing. +4. **Tier 1 MCP-tool call shape** -> Direct Python function call (`audit_openssf_baseline(...)`). No MCP server bootstrap; JSON-RPC serialization is a separate concern. +5. **Fixture metadata format** -> `parity.toml` at each fixture root, TOML-parsed. Matches Constitution III convention; no code execution; stdlib `tomllib`. + +Two governance-motivated additions surfaced from Q3: + +- FR-007a: Environment-gated dispatch, reviewer-list required, repo-level secret exposure forbidden. +- FR-007b: Operator-provided API-key inputs forbidden in MVP. +- SC-005a: Grep-verifiable: no other workflow references `secrets.ANTHROPIC_API_KEY` outside the gated Tier 2 workflow. + +## /speckit-analyze findings applied + +The 2026-08-10 analyze pass surfaced 8 findings (0 CRITICAL, 1 HIGH, 4 MEDIUM, 3 LOW). Applied remediations: + +- **HC1 (git-init in Tier 1 conftest)**: T012 updated with explicit `prepared_fixture` shape mirroring feature 026's `minimal_llm_repo_tree` pattern. Load-bearing -- without this, the Tier 1 harness invocation would fail before running any control. +- **MC1 (FR-010 missing-key test)**: T025 gains a `test_missing_api_key_raises_setup_error` subtest with both unit-level (`invoke_skill` raises) and integration-level (`run.py` subprocess exit code 3) assertions. +- **MC2 (FR-013 green-run summary)**: T013 gains a `capsys.readouterr()` capture + regex-pattern assertion on the summary line, run for EVERY test (green or red). +- **MC3 (FR-014 no product code changes)**: New task T031a creates `tests/darnit/parity/tier1/test_no_product_changes.py` that runs `git diff --name-only ...HEAD` and asserts no file under `packages/darnit/src/` or `packages/darnit-baseline/src/` is modified. Skips on local dev when no base ref is reachable. +- **MC4 (FR-015 determinism)**: T006 gains a "run compare() twice, assert byte-identical outputs" subtest. +- **LC1 (allowed-drift wildcard resolution)**: T014's allowed-drift positive cases expanded from just PENDING_LLM->WARN to all three (PENDING_LLM -> WARN | PASS | FAIL). +- **LC2 (grep portability)**: T024 replaces `subprocess.run(["grep", ...])` with pure-Python file iteration. +- **LC3 (T024/T031 redundancy)**: No action taken. T024 is the automated test; T031 is the maintainer sanity ritual. Both are cheap; keep both. + +Coverage after remediation: 30/30 requirements have >=1 task; 30/30 have >=1 test task (or manual sign-off for the two doc-shaped ones -- SC-009 via PR body, T033). diff --git a/specs/028-audit-parity-tests/contracts/parity-toml-schema.md b/specs/028-audit-parity-tests/contracts/parity-toml-schema.md new file mode 100644 index 0000000..8e01469 --- /dev/null +++ b/specs/028-audit-parity-tests/contracts/parity-toml-schema.md @@ -0,0 +1,115 @@ +# Contract: `parity.toml` Schema + +**Feature**: 028-audit-parity-tests | **Consumers**: fixture authors adding new corpus entries. + +## 1. Location + +- **PT-1**: `parity.toml` lives at the root of a fixture directory (`tests/darnit/parity/fixtures//parity.toml`). Never elsewhere. +- **PT-2**: `parity.toml` is OPTIONAL. A fixture without one still participates in inter-path parity assertions. + +## 2. Parsing + +- **PT-3**: Parsed by stdlib `tomllib.load()`. No custom TOML parser; no code execution at load time. +- **PT-4**: If `parity.toml` exists but is unparseable TOML, the fixture's Tier 1 test FAILS with a "malformed metadata" error. The fixture is not silently skipped. + +## 3. Schema + +### 3.1 `[expected]` section + +Required top-level table when `parity.toml` exists. + +```toml +[expected] +category = "mixed" # required; one of "all_pass" | "all_fail" | "mixed" | "pending_llm" +has_pending_llm = true # optional; auto-derived from counts.pending_llm > 0 if absent +strict = false # optional; default false +``` + +- **PT-5**: `category` MUST be one of the four literal strings. Any other value fails validation. +- **PT-6**: `has_pending_llm`, when explicitly set, MUST agree with `counts.pending_llm > 0` (if `counts` is present). Disagreement is a validation error. +- **PT-7**: `strict` controls whether `counts` mismatches (see below) FAIL or WARN. + +### 3.2 `[expected.counts]` sub-section + +Optional. When present, provides expected control-status distribution. + +```toml +[expected.counts] +pass = 3 +fail = 2 +warn = 1 +error = 0 +n_a = 0 +pending_llm = 1 +``` + +- **PT-8**: Every key MUST be a non-negative integer. +- **PT-9**: Unrecognized keys log a warning but do not fail validation (forward-compat). +- **PT-10**: When `strict = true`, the actual counts from a live audit MUST equal the declared counts, or the fixture's Tier 1 test FAILS. +- **PT-11**: When `strict = false` (default), a mismatch produces a non-fatal note in the pytest output (informational; useful when a control's status changes due to an upstream `openssf-baseline.toml` update). + +### 3.3 `[[expected.controls]]` array + +Optional. Per-control expectations for specific controls. + +```toml +[[expected.controls]] +id = "OSPS-GV-01.01" +status = "PASS" + +[[expected.controls]] +id = "OSPS-BR-06.01" +status = "FAIL" +``` + +- **PT-12**: `id` MUST match a control the audit produces for this fixture; otherwise validation warns. +- **PT-13**: `status` MUST be one of the six PassOutcome literals. `PENDING_LLM` is allowed here for the pending_llm category. +- **PT-14**: The actual status from BOTH the MCP tool and the harness (modulo the PENDING_LLM allowed drift) MUST equal `status`. Mismatch fails when `strict = true`; notes when `strict = false`. + +## 4. Discovery + iteration + +- **PT-15**: `tier1/fixture_meta.py` provides `load_parity_metadata(fixture_dir: Path) -> ParityMetadata | None`. Returns `None` if `parity.toml` is absent. +- **PT-16**: `test_corpus_inventory.py` (SC-008) iterates fixtures, calls `load_parity_metadata`, and counts fixtures per `category`. Passes iff at least one fixture is present in each of the four categories. + +## 5. Example: all_pass_repo + +```toml +[expected] +category = "all_pass" +has_pending_llm = false +strict = false + +[expected.counts] +pass = 8 +fail = 0 +warn = 0 +error = 0 +n_a = 4 +pending_llm = 0 +``` + +## 6. Example: pending_llm_repo + +```toml +[expected] +category = "pending_llm" +has_pending_llm = true + +[expected.counts] +pass = 4 +fail = 2 +warn = 1 +error = 0 +n_a = 5 +pending_llm = 1 + +[[expected.controls]] +id = "STAGE1-REF-SECURITY-01" +status = "PENDING_LLM" +``` + +## 7. What `parity.toml` MUST NOT be used for + +- **PT-17**: MUST NOT declare allowed drift beyond the canonical Tier 1 table (see `tier1-parity-invariant.md`). If a fixture legitimately needs a different drift class, that is a spec change, not a fixture change. +- **PT-18**: MUST NOT influence what controls run. Fixtures use `.baseline.toml` for that; `parity.toml` is test-side metadata only. +- **PT-19**: MUST NOT contain executable content, template placeholders, or references to environment variables. diff --git a/specs/028-audit-parity-tests/contracts/tier1-parity-invariant.md b/specs/028-audit-parity-tests/contracts/tier1-parity-invariant.md new file mode 100644 index 0000000..d08459d --- /dev/null +++ b/specs/028-audit-parity-tests/contracts/tier1-parity-invariant.md @@ -0,0 +1,47 @@ +# Contract: Tier 1 Parity Invariant + +**Feature**: 028-audit-parity-tests | **Consumers**: any maintainer opening a PR that touches the harness or MCP-tool code path. + +Tier 1 is a mechanical assertion about the darnit audit: two consumers of the same audit -- the MCP tool and the harness -- must produce the same per-control status modulo one documented drift class. If they don't, one of them has a bug. + +## 1. What Tier 1 asserts + +- **T1-1**: For every fixture in `tests/darnit/parity/fixtures/`, Tier 1 invokes the audit via BOTH paths (direct Python call to `darnit_baseline.tools.audit_openssf_baseline` AND `darnit.harness.driver.HarnessRun.run()` with `MockLLMStep`), normalizes both outputs to `AuditResult`, and compares per-control status. +- **T1-2**: Statuses that agree produce no drift entry. +- **T1-3**: A control that appears in one path's result but not the other is a HARD failure (test fails with a "missing control" diagnostic). +- **T1-4**: A control whose statuses differ produces a `DriftEntry`. The comparator classifies it via the allowed-drift table. +- **T1-5**: The test PASSES iff every `DriftEntry` in every fixture's comparison satisfies `is_allowed_drift == True`. +- **T1-6**: On PASS, each fixture's summary line is emitted to the pytest report per FR-013. +- **T1-7**: On FAIL, the assertion message includes a fixed-width Markdown table (no ANSI escapes) listing every disallowed drift with columns `control_id`, `mcp_status`, `harness_status`. + +## 2. Allowed drift table (canonical) + +| MCP tool status | Harness status | Verdict | +|-----------------|----------------|---------| +| Any X | Same X | agree; no drift entry produced | +| PENDING_LLM | Any non-PENDING_LLM (PASS, FAIL, WARN, N/A, ERROR) | ALLOWED DRIFT (harness's LLM continuation loop resolved) | +| PENDING_LLM | PENDING_LLM | agree; no drift entry produced | +| Any X (not PENDING_LLM) | PENDING_LLM | DISALLOWED (harness must resolve; not the other way around) | +| X | Y (X != Y; neither PENDING_LLM) | DISALLOWED | + +- **T1-8**: The table is the SOLE definition of allowed drift. Adding another row is a spec change to feature 028 (or a follow-up feature with its own spec). +- **T1-9**: The comparator implementation MUST mirror this table exactly. A unit test on the comparator enumerates every (mcp, harness) pair from the six possible statuses and asserts the correct classification. + +## 3. Runtime constraints + +- **T1-10**: The full Tier 1 suite MUST complete in under 60 seconds on a standard developer laptop (SC-002). +- **T1-11**: Each individual fixture's parity check SHOULD complete in under 10 seconds. A fixture that consistently exceeds 10s is a bug against this contract; the fixture is either too large or triggers a slow control. +- **T1-12**: Tier 1 MUST NOT make any network call, live LLM API call, or subprocess spawn. `MockLLMStep` is the sole LLM-side seam. Any test that observes a network call in Tier 1 is a bug against this contract. +- **T1-13**: Tier 1 MUST be deterministic. Repeated runs against an unchanged fixture MUST produce identical drift verdicts. If a Tier 1 test is flaky, the flake is a bug (either in the test or in an audit path that has time-of-day sensitivity). + +## 4. Adversarial-test coverage + +- **T1-14**: A test in `test_comparator_adversarial.py` seeds a hand-built diverging `AuditResult` pair (PASS vs FAIL on the same control) and asserts the comparator flags it as a disallowed drift. This is SC-001's mechanical verification. +- **T1-15**: A test in `test_comparator_adversarial.py` seeds N (>=3) divergences in one comparison and asserts the failure message contains exactly N rows in its drift table. This is SC-003's mechanical verification. + +## 5. What Tier 1 does NOT assert + +- **T1-16**: Does not check that the audit is "correct" in any absolute sense. Only that the two paths agree. +- **T1-17**: Does not exercise interactive answer collection (feature 027 territory). +- **T1-18**: Does not exercise re-audit-on-fresh-answer behavior (deferred; feature 026's MVP no-reaudit policy still holds). +- **T1-19**: Does not compare authority-level values across paths beyond the status field. Authority is verified elsewhere (feature 025 T005/T014/T016). diff --git a/specs/028-audit-parity-tests/contracts/tier2-workflow.md b/specs/028-audit-parity-tests/contracts/tier2-workflow.md new file mode 100644 index 0000000..f083dee --- /dev/null +++ b/specs/028-audit-parity-tests/contracts/tier2-workflow.md @@ -0,0 +1,74 @@ +# Contract: Tier 2 Workflow Configuration + Governance + +**Feature**: 028-audit-parity-tests | **Consumers**: maintainers configuring the GitHub Actions Environment, reviewers approving Tier 2 dispatches, auditors verifying access-control compliance. + +Tier 2 is a manual-dispatch-only GitHub Actions workflow that invokes the `/darnit-audit` coding-agent skill via the Claude Agent SDK, diffs against the raw MCP tool output, and captures the drift. The workflow's security posture is the load-bearing property. + +## 1. Trigger + Environment + +- **T2-1**: The workflow is triggered EXCLUSIVELY by `workflow_dispatch`. No `push`, no `pull_request`, no `schedule` trigger. +- **T2-2**: The workflow's job MUST declare `environment: parity-tier2` (name is the exact string; case-sensitive). +- **T2-3**: The GitHub Environment `parity-tier2` MUST be configured (via GitHub UI, not via YAML in the repo) with: + - A required-reviewers list of authorized maintainers. + - The `ANTHROPIC_API_KEY` secret stored AT THE ENVIRONMENT LEVEL, not at the repository level. +- **T2-4**: No other workflow file in `.github/workflows/` references `secrets.ANTHROPIC_API_KEY`. Verifiable via `grep -r 'ANTHROPIC_API_KEY' .github/workflows/ | grep -v parity-tier2.yml` returning zero lines. This is SC-005a's assertion. + +## 2. Permissions + +- **T2-5**: The workflow job MUST declare `permissions: contents: read` at the job level. No `write` scope is granted to any resource. +- **T2-6**: The workflow MUST NOT use any Action that requires a token with elevated scope (e.g., no `peter-evans/create-pull-request`). + +## 3. Preflight audit + +- **T2-7**: Before the SDK invocation step, the workflow MUST log to `GITHUB_STEP_SUMMARY`: + - The actor (`github.actor`) who triggered the dispatch. + - The commit SHA. + - The exact `fixture_glob` input value. + - The current wall-clock timestamp. +- **T2-8**: The preflight step MUST be BEFORE the step that consumes `ANTHROPIC_API_KEY`, so a post-hoc audit can attribute cost even if the API call itself fails. + +## 4. Input parameters + +- **T2-9**: The workflow accepts exactly one input, `fixture_glob`, defaulting to `"*"`. It filters which fixtures under `tests/darnit/parity/fixtures/` are exercised. +- **T2-10**: The workflow MUST NOT accept an API key as an input parameter (per FR-007b). Adding one is a governance regression. + +## 5. Artifact upload + +- **T2-11**: On any exit code (0 or non-zero), the workflow MUST upload the contents of `parity-artifacts/` via `actions/upload-artifact`. Failure to upload is itself a workflow failure. +- **T2-12**: Artifact retention MUST NOT exceed 90 days by default (Anthropic API calls have IP addresses / timestamps in their transcripts; long retention increases surface). + +## 6. Exit codes + reporting + +- **T2-13**: The runner Python script (`tests/darnit/parity/tier2/run.py`) exits with: + - `0` iff every fixture's skill output agrees with the raw tool output on per-control status. + - `1` if any fixture has a skill-vs-tool disagreement. + - `2` if any fixture's skill output was unparseable (distinct from disagreement). + - `3` for setup errors (missing key, missing fixtures, SDK import failure). + - `4` for rate-limit exhaustion (partial results captured, artifacts uploaded, workflow fails). +- **T2-14**: A summary of the failure classes is written to `GITHUB_STEP_SUMMARY` even on success, so the auditor sees "N fixtures checked, 0 drifts" as evidence. + +## 7. Rate limit + retry + +- **T2-15**: The runner MUST NOT retry API calls automatically. A rate-limit hit is a documented failure (`exit 4`) with instructions in the summary to re-dispatch manually. +- **T2-16**: Each fixture MUST be exercised at most once per workflow invocation. Multiple invocations to work around a rate limit are the maintainer's manual choice. + +## 8. What Tier 2 does NOT do + +- **T2-17**: Does not automatically remediate any drift. Diagnostic only (FR-016). +- **T2-18**: Does not modify the `/darnit-audit` skill under any circumstance. +- **T2-19**: Does not modify any darnit product package. +- **T2-20**: Does not attempt to run without `ANTHROPIC_API_KEY`; silent skip is forbidden (FR-010). + +## 9. Test coverage of the workflow itself + +- **T2-21**: A test at `tests/darnit/parity/tier2/test_workflow_config.py` (Tier 1 -- offline) parses `.github/workflows/parity-tier2.yml` as YAML and asserts: + - Only trigger is `workflow_dispatch`. + - Job declares `environment: parity-tier2`. + - Job declares `permissions: contents: read`. + - `ANTHROPIC_API_KEY` is only referenced under an `env:` block in the SDK-invocation step. +- **T2-22**: A test in the same file greps `.github/workflows/` for `ANTHROPIC_API_KEY` references outside `parity-tier2.yml` and asserts the count is zero. + +## 10. Governance escalation + +- **T2-23**: If the workflow YAML is modified in a way that removes any of T2-1 through T2-6 (trigger, environment, permissions, secret scope), that PR MUST be reviewed by TWO maintainers (a two-person integrity rule for security-critical config). This is enforced via CODEOWNERS or branch protection rather than YAML. +- **T2-24**: A modification to `.github/workflows/parity-tier2.yml` in the same PR as an unrelated feature is a code smell and SHOULD be split into a dedicated PR for review clarity. diff --git a/specs/028-audit-parity-tests/data-model.md b/specs/028-audit-parity-tests/data-model.md new file mode 100644 index 0000000..52d235d --- /dev/null +++ b/specs/028-audit-parity-tests/data-model.md @@ -0,0 +1,220 @@ +# Phase 1 Data Model: Two-Tier Audit Parity Tests + +**Feature**: 028-audit-parity-tests | **Date**: 2026-08-09 + +All entities in this feature are TEST-side only. No production data model changes. + +## 1. `Fixture` + +Location on disk: `tests/darnit/parity/fixtures//` + +Not a Python class; represented by a filesystem directory. A directory is "a fixture" iff it satisfies: + +- Contains a `.baseline.toml` at its root (required). +- Optionally contains `.project/project.yaml` for context values. +- Optionally contains a `parity.toml` at its root declaring expected shape (see section 2). +- May contain any other repo files the controls reference (LICENSE, README, `.github/`, etc.). + +**Identity**: the directory name (e.g. `all_pass_repo`). Used as the pytest test ID. + +**Discovery**: `tests/darnit/parity/tier1/conftest.py` iterates `tests/darnit/parity/fixtures/` and yields each directory that contains `.baseline.toml`. + +## 2. `parity.toml` schema (fixture metadata) + +Location: `tests/darnit/parity/fixtures//parity.toml` + +Optional per-fixture file. Absence is allowed; a fixture without `parity.toml` still participates in inter-path parity assertions but is excluded from corpus-inventory checks (SC-008). + +```toml +[expected] +# One of "all_pass", "all_fail", "mixed", "pending_llm". +# Used by SC-008 corpus-inventory check. +category = "mixed" + +# Expected control-status distribution when running the audit via EITHER +# the MCP tool or the harness (they must agree modulo the PENDING_LLM +# allowed drift). Purely informational for regression clarity; not +# enforced against runtime counts unless `strict = true`. +[expected.counts] +pass = 3 +fail = 2 +warn = 1 +error = 0 +n_a = 0 +# Number of controls the MCP tool leaves PENDING_LLM. The harness +# resolves these; harness's `warn` may be higher than tool's by this many. +pending_llm = 1 + +# Optional flags +[expected] +has_pending_llm = true # true if pending_llm > 0 +strict = false # if true, counts are enforced (mismatch fails); + # if false, counts are advisory (mismatch logs a note) + +# Optional per-control expectations. Rarely used; when present, overrides +# aggregate count checks for the named controls. +[[expected.controls]] +id = "OSPS-GV-01.01" +status = "PASS" # what BOTH paths must report +``` + +**Validation** (`tier1/fixture_meta.py`): + +- Parses via stdlib `tomllib`. +- `category` MUST be one of the four literals. +- `counts.*` must be non-negative integers. +- `has_pending_llm` MUST match `counts.pending_llm > 0`. +- Unknown keys log a warning but do not fail (forward-compatibility). + +## 3. `AuditResult` (normalized comparison target) + +Module: `tests/darnit/parity/tier1/comparator.py` + +```python +@dataclass(frozen=True) +class Control: + id: str + status: Literal["PASS", "FAIL", "WARN", "N/A", "ERROR", "PENDING_LLM"] + authority: Literal["dispositive", "suggestive", "asserted"] | None + level: int | None + +@dataclass(frozen=True) +class AuditResult: + """Normalized shape both the MCP tool JSON and the harness report + reduce to for comparison.""" + + controls: tuple[Control, ...] + source: Literal["mcp_tool", "harness"] + + @classmethod + def from_mcp_json(cls, payload: dict) -> AuditResult: + """Parse the JSON output of audit_openssf_baseline(output_format='json').""" + ... + + @classmethod + def from_harness_report(cls, report: HarnessReport) -> AuditResult: + """Reduce a HarnessReport (feature 026 model) to the same shape.""" + ... +``` + +**Notes**: + +- The MCP tool returns a JSON string with a top-level `results` list; each result has `id`, `status`, `authority`, `level` at minimum. `from_mcp_json` picks those four fields. +- The harness's `HarnessReport.controls` is a list of dicts with the same shape. `from_harness_report` picks the same four. +- Frozen dataclass so `AuditResult` instances are hashable + immutable across the comparator's iterations. + +## 4. `DriftEntry` (one comparison row) + +Module: `tests/darnit/parity/tier1/comparator.py` + +```python +@dataclass(frozen=True) +class DriftEntry: + fixture_name: str + control_id: str + mcp_status: str + harness_status: str + + @property + def is_allowed_drift(self) -> bool: + """PENDING_LLM -> any non-PENDING_LLM is allowed (R2). + Any other status difference is disallowed. + Statuses that agree don't produce DriftEntry at all.""" + if self.mcp_status == "PENDING_LLM" and self.harness_status != "PENDING_LLM": + return True + return False +``` + +## 5. `ParityReport` (comparator output) + +Module: `tests/darnit/parity/tier1/comparator.py` + +```python +@dataclass(frozen=True) +class ParityReport: + fixture_name: str + total_controls: int + agreements: int + drifts: tuple[DriftEntry, ...] + + @property + def disallowed_drifts(self) -> tuple[DriftEntry, ...]: + return tuple(d for d in self.drifts if not d.is_allowed_drift) + + @property + def is_green(self) -> bool: + return len(self.disallowed_drifts) == 0 + + def format_summary_line(self) -> str: + """FR-013 evidence line, emitted on every run.""" + allowed = sum(1 for d in self.drifts if d.is_allowed_drift) + return ( + f"[tier1] {self.fixture_name}: " + f"{self.total_controls} controls compared, " + f"{self.agreements} agreed, " + f"{len(self.disallowed_drifts)} diverged, " + f"{allowed} allowed-drift" + ) + + def format_failure_table(self) -> str: + """Only called when there are disallowed drifts. Produces a + fixed-width Markdown table (no ANSI) for pytest assertion messages.""" + ... +``` + +**FR-004 shape**: `format_failure_table` produces something like: + +``` +| control_id | mcp_status | harness_status | +|-----------------------|------------|----------------| +| OSPS-GV-01.01 | PASS | FAIL | +| OSPS-BR-06.01 | FAIL | WARN | +``` + +## 6. `SkillReport` (Tier 2 parsed skill output) + +Module: `tests/darnit/parity/tier2/skill_markdown_parser.py` + +```python +@dataclass(frozen=True) +class SkillReport: + parseable: bool + raw_markdown: str # always populated + counts: dict[str, int] | None # {"pass": 51, "fail": 5, ...} or None if unparseable + controls: tuple[Control, ...] | None # per-control claims or None if unparseable + parse_notes: tuple[str, ...] # human-readable notes on best-effort extractions + + @classmethod + def parse(cls, markdown: str) -> SkillReport: + """Best-effort regex parser. Never raises; sets parseable=False + instead of failing.""" + ... +``` + +**Failure classification**: + +- `parseable == False` -> Tier 2 emits "skill output unparseable" verdict; NOT "skill and tool disagree." Distinct failure class per FR-006a. +- `parseable == True` but counts differ from tool -> "counts disagree" verdict. +- `parseable == True` and per-control claims differ from tool -> "per-control disagree" verdict (strongest, includes the offending control IDs in the failure artifact). + +## 7. Tier 2 artifact bundle + +For every fixture Tier 2 exercises (whether pass or fail), the workflow writes: + +``` +parity-artifacts/ ++-- / + +-- mcp_tool_result.json # Raw stringified JSON from audit_openssf_baseline + +-- skill_final_message.md # The final assistant message from the Agent SDK invocation + +-- diff_report.md # Human-readable diff (pass or fail); FR-009 requirement + +-- metadata.json # invocation timestamp, actor, git SHA, model ID, turn count +``` + +`parity-artifacts/` is uploaded via `actions/upload-artifact` at end of job. Retention: 30 days by default (a GitHub setting; not spec-controlled). + +## 8. Relationship to existing product entities + +- **Consumer of `darnit_baseline.tools.audit_openssf_baseline`**: Tier 1 imports and calls; Tier 2 imports and calls before invoking the SDK. +- **Consumer of `darnit.harness.driver.HarnessRun`**: Tier 1 constructs with `MockLLMStep`; never Tier 2. +- **Consumer of `darnit.core.llm_step.MockLLMStep`**: Tier 1 only. +- **No modification of `HarnessRun`, `HarnessReport`, `audit_openssf_baseline`, or any product model.** SC-006 hard rule. diff --git a/specs/028-audit-parity-tests/plan.md b/specs/028-audit-parity-tests/plan.md new file mode 100644 index 0000000..db57ff1 --- /dev/null +++ b/specs/028-audit-parity-tests/plan.md @@ -0,0 +1,139 @@ +# Implementation Plan: Two-Tier Audit Parity Tests + +**Branch**: `028-audit-parity-tests` | **Date**: 2026-08-09 | **Spec**: [spec.md](spec.md) + +**Input**: Feature specification from `specs/028-audit-parity-tests/spec.md` (with 5 clarifications from `/speckit-clarify` on 2026-08-09: Claude Agent SDK for Tier 2 invocation; skill's final assistant message as the parsed artifact; manual-dispatch only with Environment-gated key + reviewer approval; direct Python function call for Tier 1; `parity.toml` per fixture). + +## Summary + +Adds a two-tier diagnostic test suite that verifies the darnit audit's per-control output is consistent across the three consumers users care about: + +- **Direct MCP tool** call (`darnit_baseline.tools.audit_openssf_baseline`) +- **`darnit harness`** end-to-end +- **`/darnit-audit` coding-agent skill** (Tier 2 only) + +**Tier 1** (`tests/darnit/parity/tier1/`) is a pytest suite that runs on every PR. For each fixture in `tests/darnit/parity/fixtures/`, it invokes the MCP tool directly AND runs the harness in-process (both with `MockLLMStep` -- no live API), then diffs their per-control status. The sole allowed drift is: the MCP tool leaves a control PENDING_LLM; the harness resolves it to any non-PENDING_LLM status via its LLM continuation loop. Anything else is a hard failure with a human-readable diff table. + +**Tier 2** (`tests/darnit/parity/tier2/`) is a manual-dispatch-only GitHub Actions workflow. For each fixture, it captures the raw MCP tool JSON and invokes the `/darnit-audit` coding-agent skill via the Claude Agent SDK on the same fixture, then diffs the skill's final assistant message against the raw tool output. Any per-control status difference is a hard failure regardless of authority level. Access control (Environment-gated `ANTHROPIC_API_KEY` + required-reviewer approval) prevents unauthorized dispatches from spending API budget. + +**Fixture corpus** starts with the existing `minimal_llm_repo` (reused from feature 026's fixture tree) plus new synthetic fixtures for the all-PASS, all-FAIL, mixed, and PENDING_LLM shapes. Each fixture optionally carries a `parity.toml` declaring its expected shape (TOML-parsed via stdlib `tomllib`); corpus-inventory checks (SC-008) use this file. + +Closes #366. Diagnostic only -- any drift Tier 2 discovers becomes a separate feature to fix. Follow-up issues #368 (OpenAI SDK + other-provider parity) and #369 (scheduled cadence + governance-appropriate key sourcing) capture explicitly out-of-scope work. + +## Technical Context + +**Language/Version**: Python 3.11 / 3.12 (workspace targets, unchanged). + +**Primary Dependencies (new -- test-side only, no product impact)**: + +- `claude-agent-sdk` (Anthropic's Python SDK for scripted agent invocations). Added to a new `tests/darnit/parity/pyproject.toml` as a dev-group dep, NOT to any product package's `pyproject.toml`. SC-006 hard requirement. +- Nothing else new; `tomllib` is stdlib. + +**Primary Dependencies (in use)**: `pydantic >= 2.0` (existing), `pytest` (existing), feature 026's `HarnessRun` + `MockLLMStep`, feature 025's `LLMStep` Protocol, `darnit_baseline.tools.audit_openssf_baseline`. + +**Storage**: Filesystem only. Fixture directories under `tests/darnit/parity/fixtures/`. Tier 2 output artifacts (skill Markdown + tool JSON on failure) written to `parity-artifacts/` at CI job root, uploaded as workflow artifacts. No new persistent state. + +**Testing**: pytest for Tier 1. Tier 2's runner is a Python script (`tests/darnit/parity/tier2/run.py`) invoked from GitHub Actions; not pytest because it hits a live API and needs artifact-write semantics that don't fit the pytest lifecycle cleanly. + +**Target Platform**: Any host that runs darnit tests (Linux, macOS) for Tier 1. Tier 2 runs on GitHub-hosted `ubuntu-latest` in the workflow-dispatch job. + +**Project Type**: Test suite + one GitHub Actions workflow. No product code changes. + +**Performance Goals**: Tier 1 -- full corpus in under 60s (SC-002). Individual fixture in under 10s. Tier 2 -- one skill invocation per fixture; the workflow's total wall time depends on Anthropic API latency (typically 30s-120s per skill run for a small repo). + +**Constraints**: + +- **SC-006 hard rule**: no new runtime deps on `packages/darnit/pyproject.toml` or `packages/darnit-baseline/pyproject.toml`. All new deps live under a test-only dev group. +- **FR-014**: no product code changes. If a Tier-1 test needs a helper that doesn't exist on the harness or MCP tool, we flag it as follow-up work, we do not silently add product code. +- **FR-007a governance**: `ANTHROPIC_API_KEY` MUST live in a GitHub Environment (not a repository secret) and be reachable ONLY from the gated Tier 2 workflow. SC-005a is grep-verifiable. +- **FR-003**: Tier 1 offline. `MockLLMStep` for the harness; never a live API call. + +**Scale/Scope**: MVP corpus is 4-6 fixtures. Small test suite (~600-800 lines total: ~300 Tier 1 test + comparator + auto-discovery machinery, ~200 Tier 2 runner + SDK invocation, ~100 skill Markdown parser, ~200 test fixtures). No new modules in `packages/`. + +## Constitution Check + +Constitution v1.3.0. Five Core Principles evaluated as gates. + +| Principle | Applicable? | Verdict | Rationale | +|-----------|-------------|---------|-----------| +| I. Plugin Separation | Yes | PASS | The parity tests are consumers, not part of `darnit-core` or `darnit-baseline`. They import both packages' public surfaces (MCP tool function; harness `HarnessRun`) but add no code TO either package. Tier 2's Claude Agent SDK dep is test-only per FR-006 + SC-006. | +| II. Conservative-by-Default | Yes | PASS + REINFORCED | This feature exists to protect conservatism: the whole point of Tier 2 is catching the skill silently reclassifying a WARN as PASS. If the skill layer erodes the "WARN counts as FAIL" property from Principle II, Tier 2 makes the erosion visible. Tier 1 catches equivalent regressions in the harness before they merge. | +| III. TOML-First Architecture | Yes | PASS | Fixture metadata uses TOML (`parity.toml`), matching the framework's control-config format. No control changes; no new schema fields on framework TOMLs. Just fixture-side test metadata. | +| IV. Never Guess User Values | Yes | PASS + REINFORCED | Fixtures do NOT auto-generate context values. Any value a fixture needs is written explicitly in its `.project/project.yaml` at fixture-authoring time; the parity tests read whatever's there. No heuristic value inference. Related to Principle IV: Tier 2 exists precisely because a downstream layer (the coding-agent skill) was silently applying a heuristic interpretation of the tool's output; this feature makes those heuristics visible so the constitution's guarantee is externally observable. | +| V. Sieve Pipeline Integrity | Yes | PASS (N/A in substance) | This feature is downstream of the sieve; it does not modify the 4-phase pipeline. It exercises the same `run_sieve_audit` seam through two consumers to verify they produce identical output. | + +**No violations.** No Complexity Tracking entries required. + +Governance observations (not constitution violations, but worth calling out): + +- FR-007 through FR-007b + SC-005a locks down a real risk: an unauthorized community member spending money out of a company-owned API key. This is enforced at the GitHub Actions Environment layer (not in code), so the security depends on correct workflow configuration. The plan's contract MUST include a workflow-config review step before merge. +- Feature 026's "no re-audit after collect" MVP policy is untouched. Neither Tier 1 nor Tier 2 exercises interactive answer collection (that's feature 027 territory); the resolver chain stays empty. + +## Project Structure + +### Documentation (this feature) + +```text +specs/028-audit-parity-tests/ ++-- spec.md # /speckit-specify + /speckit-clarify output ++-- plan.md # this file ++-- research.md # Phase 0: architectural decisions ++-- data-model.md # Phase 1: Fixture, AuditResult, DriftEntry, SkillReport, ParityReport ++-- quickstart.md # Phase 1: how to run Tier 1 locally + Tier 2 via workflow_dispatch ++-- contracts/ +| +-- tier1-parity-invariant.md # What Tier 1 asserts + allowed-drift table +| +-- tier2-workflow.md # workflow_dispatch shape + Environment/reviewer/secret config +| +-- parity-toml-schema.md # `parity.toml` schema fixtures may declare ++-- checklists/ +| +-- requirements.md # spec-quality checklist (exists) ++-- tasks.md # /speckit-tasks output (later) +``` + +### Source Code (repository root) + +**Zero changes to `packages/darnit/` or `packages/darnit-baseline/`.** Everything ships under `tests/` and `.github/workflows/`. + +```text +tests/darnit/parity/ ++-- __init__.py # empty ++-- fixtures/ # NEW: the corpus lives here +| +-- all_pass_repo/ +| | +-- .baseline.toml # fixture config +| | +-- .project/project.yaml # explicit context values +| | +-- parity.toml # {[expected] category="all_pass", counts={...}} +| | +-- # LICENSE, README, etc as needed +| +-- all_fail_repo/ +| | +-- .baseline.toml +| | +-- parity.toml # {[expected] category="all_fail", ...} +| | +-- +| +-- mixed_repo/ +| | +-- ... # some pass, some fail, some warn +| | +-- parity.toml # {[expected] category="mixed", ...} +| +-- pending_llm_repo/ # can reuse minimal_llm_repo from feature 026 +| +-- ... +| +-- parity.toml # {[expected] category="pending_llm", has_pending_llm=true} ++-- tier1/ +| +-- __init__.py +| +-- conftest.py # fixture auto-discovery pytest plugin +| +-- comparator.py # AuditResult diff logic + DriftEntry construction + table formatting +| +-- fixture_meta.py # parity.toml parser + schema validation +| +-- test_mcp_vs_harness.py # parametrized-per-fixture parity assertions +| +-- test_corpus_inventory.py # SC-008: assert at least one fixture per category +| +-- test_comparator_adversarial.py # SC-001/003: seeds divergences, asserts they're caught ++-- tier2/ + +-- __init__.py + +-- run.py # entrypoint; invoked from workflow_dispatch + +-- skill_markdown_parser.py # best-effort parser for skill's final assistant message + +-- claude_agent_sdk_client.py # thin wrapper around the SDK; deterministic invocation + +-- artifact_writer.py # writes tool JSON + skill Markdown to parity-artifacts/ + +-- diff.py # per-control status comparison, produces failure report + +.github/workflows/ ++-- parity-tier2.yml # NEW: workflow_dispatch-triggered; Environment-gated +``` + +**Structure Decision**: Test-suite-only. No `packages/` changes anywhere. The Claude Agent SDK dep is declared in a new dependency group in the workspace-level `pyproject.toml` (dev group) so `uv sync --dev` installs it for maintainers but no downstream user of darnit-core / darnit-baseline gets it as a transitive dep. + +## Complexity Tracking + +No violations. Section intentionally empty. diff --git a/specs/028-audit-parity-tests/quickstart.md b/specs/028-audit-parity-tests/quickstart.md new file mode 100644 index 0000000..9dc9bd1 --- /dev/null +++ b/specs/028-audit-parity-tests/quickstart.md @@ -0,0 +1,166 @@ +# Quickstart: Two-Tier Audit Parity Tests + +**Feature**: 028-audit-parity-tests | **For**: maintainers running Tier 1 locally on a PR, or authorized reviewers dispatching Tier 2 to check for coding-agent skill drift. + +## Tier 1: run locally on every PR + +```bash +# Full suite (fast; no live API) +uv run pytest tests/darnit/parity/tier1/ -q + +# Single fixture +uv run pytest tests/darnit/parity/tier1/test_mcp_vs_harness.py -q -k "mixed_repo" + +# See the per-fixture summary lines even on green +uv run pytest tests/darnit/parity/tier1/ -v -s +``` + +Expected on a green run: + +``` +tests/darnit/parity/tier1/test_mcp_vs_harness.py::test_parity[all_pass_repo] PASSED +tests/darnit/parity/tier1/test_mcp_vs_harness.py::test_parity[all_fail_repo] PASSED +tests/darnit/parity/tier1/test_mcp_vs_harness.py::test_parity[mixed_repo] PASSED +tests/darnit/parity/tier1/test_mcp_vs_harness.py::test_parity[pending_llm_repo] PASSED + +[tier1] all_pass_repo: 8 controls compared, 8 agreed, 0 diverged, 0 allowed-drift +[tier1] all_fail_repo: 12 controls compared, 12 agreed, 0 diverged, 0 allowed-drift +[tier1] mixed_repo: 14 controls compared, 14 agreed, 0 diverged, 0 allowed-drift +[tier1] pending_llm_repo: 15 controls compared, 14 agreed, 0 diverged, 1 allowed-drift (PENDING_LLM->WARN) +``` + +Expected on a failure (harness silently disagreeing with the tool on OSPS-GV-01.01): + +``` +FAILED tests/darnit/parity/tier1/test_mcp_vs_harness.py::test_parity[mixed_repo] + +Assertion: harness disagrees with MCP tool beyond documented allowed drift. + +| control_id | mcp_status | harness_status | +|----------------|------------|----------------| +| OSPS-GV-01.01 | PASS | FAIL | +``` + +## Adding a new fixture + +```bash +# 1. Create the fixture directory + files +mkdir -p tests/darnit/parity/fixtures/my_new_fixture +cd tests/darnit/parity/fixtures/my_new_fixture +touch .baseline.toml +# ... populate with the file set your controls exercise + +# 2. Run the audit to capture expected counts +uv run python -c " +from darnit_baseline.tools import audit_openssf_baseline +import json +result = json.loads(audit_openssf_baseline(local_path='.', level=1, output_format='json')) +counts = {} +for c in result['results']: + counts[c['status'].lower()] = counts.get(c['status'].lower(), 0) + 1 +print(counts) +" + +# 3. Write parity.toml with the captured counts +cat > parity.toml <<'EOF' +[expected] +category = "mixed" + +[expected.counts] +pass = 4 +fail = 2 +warn = 1 +error = 0 +n_a = 3 +pending_llm = 0 +EOF + +# 4. Verify the new fixture is picked up +uv run pytest tests/darnit/parity/tier1/ -q -k "my_new_fixture" +``` + +No test file changes needed -- the fixture is auto-discovered by `conftest.py`. + +## Tier 2: dispatch via GitHub Actions (authorized reviewers only) + +Tier 2 is **NOT** run automatically. It is triggered manually by an authorized maintainer. + +```bash +# From gh CLI (requires push access to the repo) +gh workflow run parity-tier2.yml --repo darnitdevorg/darnit -f fixture_glob="*" + +# Or from the GitHub UI: +# Actions -> "Parity Tier 2" -> Run workflow -> approve when prompted +``` + +Because the workflow uses a required-reviewer Environment, the run will PAUSE at the approval gate until an authorized reviewer clicks "Approve." Only then does the `ANTHROPIC_API_KEY` become available to the job. + +### On success (workflow exits 0) + +- Green check on the workflow run. +- `parity-artifacts/` uploaded as a workflow artifact for every fixture. +- Summary in the job's `GITHUB_STEP_SUMMARY`: + + ``` + Tier 2 parity check: 4 fixtures checked, 0 drifts, 0 unparseable, 0 rate-limited + ``` + +### On failure + +Exit codes tell you the failure class: + +- `exit 1`: skill and tool disagree on per-control status. `parity-artifacts//diff_report.md` shows which control(s). +- `exit 2`: skill output couldn't be parsed. Inspect `parity-artifacts//skill_final_message.md` to see what the skill produced. +- `exit 3`: setup error (missing key, missing fixture, SDK import failure). Check the workflow logs. +- `exit 4`: rate limit exhausted mid-run. Partial results in artifacts; re-dispatch later. + +### Reviewing artifacts locally + +```bash +gh run download --repo darnitdevorg/darnit +cd parity-artifacts/mixed_repo/ +cat mcp_tool_result.json | jq '.results[] | select(.id == "OSPS-GV-01.01")' +cat skill_final_message.md +cat diff_report.md +``` + +## What Tier 1 catches vs. what Tier 2 catches + +| Regression | Tier 1 catches? | Tier 2 catches? | +|---|---|---| +| Harness silently disagrees with MCP tool on a control | YES (every PR) | YES (nightly if scheduled) | +| MCP tool changes its output format | YES (harness would break too) | YES | +| Coding-agent skill silently reclassifies WARN as PASS | NO | YES | +| Coding-agent skill's summary counts drift from tool's raw counts | NO | YES | +| An update to Claude Sonnet changes how the skill summarizes | NO | YES (over time) | +| Harness's LLM continuation loop produces different verdicts than before | YES (deterministic under MockLLMStep) | (partially -- if the change also affects live-LLM behavior) | + +Tier 1 catches product-code regressions in the harness or MCP tool. +Tier 2 catches presentation-layer regressions (or model updates) in the coding-agent skill. + +## Environment configuration (one-time, for maintainers) + +Not part of the code; done in GitHub UI. Required before Tier 2 works: + +1. Go to Settings -> Environments in the darnit repo. +2. Create Environment `parity-tier2`. +3. Add required reviewers (list of maintainers authorized to approve Tier 2 dispatches). +4. Add secret `ANTHROPIC_API_KEY` at the ENVIRONMENT level (not repo level). +5. Confirm: `Settings -> Secrets and variables -> Actions` does NOT contain `ANTHROPIC_API_KEY` at the repo level. If it does, it was misconfigured; delete the repo-level secret to enforce Environment-only scope (FR-007a). + +## Running Tier 1 in CI + +Tier 1 is included in the standard test workflow: + +```yaml +# .github/workflows/test.yml (existing) +- name: Run tests + run: uv run pytest tests/ -q +``` + +No new workflow needed for Tier 1; it's part of the default pytest run. + +## Related follow-up work + +- **Issue #368**: OpenAI SDK + other-provider parity checks (Tier 2 style, different provider). +- **Issue #369**: Scheduled Tier 2 cadence + governance-appropriate key sourcing (requires this feature to merge first). diff --git a/specs/028-audit-parity-tests/research.md b/specs/028-audit-parity-tests/research.md new file mode 100644 index 0000000..487c8a7 --- /dev/null +++ b/specs/028-audit-parity-tests/research.md @@ -0,0 +1,268 @@ +# Phase 0 Research: Two-Tier Audit Parity Tests + +**Feature**: 028-audit-parity-tests | **Date**: 2026-08-09 + +The five load-bearing decisions were resolved in `/speckit-clarify` (recorded in `spec.md`'s Clarifications block). This file covers the residual technical decisions Phase 1 needs to sit on. + +## R1. Fixture layout + auto-discovery mechanism + +**Decision**: A fixture is any directory directly under `tests/darnit/parity/fixtures/` that contains a `.baseline.toml`. Auto-discovery uses pytest parametrization via a `conftest.py` in `tests/darnit/parity/tier1/`: + +```python +# conftest.py sketch +def pytest_generate_tests(metafunc): + if "fixture_dir" in metafunc.fixturenames: + root = Path(__file__).parent.parent / "fixtures" + fixtures = [d for d in sorted(root.iterdir()) + if d.is_dir() and (d / ".baseline.toml").exists()] + metafunc.parametrize("fixture_dir", fixtures, ids=[f.name for f in fixtures]) +``` + +`test_mcp_vs_harness.py` accepts `fixture_dir: Path` and gets one test invocation per fixture, with the fixture's directory name in the test id. Adding a new fixture is a pure directory addition; no test file change. + +**Rationale**: pytest's `pytest_generate_tests` is the standard mechanism for this shape. Test IDs match directory names, so a failure in `test_mcp_vs_harness[mixed_repo]` is self-documenting. + +**Alternatives considered**: + +- Static test-function-per-fixture (one `def test_mixed_repo():`): rejected -- adding a fixture requires editing test code, violating FR-012. +- Runtime discovery via `pytest.mark.parametrize` with a module-level list computed at import time: works but harder to override during adversarial tests (see R5). + +## R2. Sole allowed drift = PENDING_LLM to any non-PENDING_LLM + +**Decision**: The comparator (`tier1/comparator.py`) implements this rule as a table: + +``` +Direct-MCP status | Harness status | Verdict +PASS | PASS | ok +FAIL | FAIL | ok +WARN | WARN | ok +N/A | N/A | ok +ERROR | ERROR | ok +PENDING_LLM | * | allowed_drift +* | PENDING_LLM | FAIL (harness must resolve; not the other way) +X | Y (X != Y) | FAIL (any other mismatch) +``` + +A separate helper on `DriftEntry` (`is_allowed_drift`) implements this table. The failure message includes both an "unallowed drifts" table (the hard failures) AND an "allowed drifts" note (for evidence, does not cause failure). This mirrors feature 026's habit of surfacing evidence even on green runs. + +**Rationale**: Explicit rule table beats scattered conditionals. Easy to add rows later if a new class of drift becomes legitimate. + +**Alternatives considered**: + +- Symmetric wildcard (`PENDING_LLM <-> *` both ways): rejected -- if the harness produced PENDING_LLM while the MCP tool resolved it, that's a real bug (the harness has an LLM continuation loop; the MCP tool doesn't). +- Configurable per-fixture drift allowances via `parity.toml`: rejected as scope creep. If a specific fixture legitimately needs a different drift class, that's a spec change that adds a new allowed row to the table. + +## R3. Harness invocation without live API + +**Decision**: Tier 1 invokes the harness via direct instantiation: + +```python +from darnit.core.llm_step import MockLLMStep, LLMJudgment +from darnit.harness.driver import HarnessRun + +async def run_harness_on_fixture(fixture_dir: Path) -> HarnessReport: + mock = MockLLMStep(LLMJudgment( + outcome="inconclusive", # never let the LLM "resolve" beyond WARN + confidence=0.0, + reasoning="Tier 1 mock -- no LLM decision", + )) + run = HarnessRun( + local_path=str(fixture_dir), + level=3, + llm_step=mock, + per_call_timeout_s=5, + total_run_timeout_s=30, + ) + return await run.run() +``` + +The mock returns `inconclusive` so any PENDING_LLM control resolves to WARN (per the harness's `verify_with_llm_response` fallthrough). That's exactly the allowed-drift class in R2. + +**Rationale**: MockLLMStep is feature 026's test seam; Tier 1 uses it exactly as feature 026's own tests do. Deterministic + fast + offline. + +**Alternatives considered**: + +- Configure the mock per-fixture to return specific outcomes: rejected as scope creep. Tier 1 verifies the paths agree on IDENTICAL inputs; simulating different LLM verdicts is a different test surface (Tier 2's territory). + +## R4. MCP tool invocation shape + +**Decision**: Tier 1 calls the MCP tool as a plain Python function: + +```python +from darnit_baseline.tools import audit_openssf_baseline +import json + +def run_mcp_tool_on_fixture(fixture_dir: Path) -> AuditResult: + raw = audit_openssf_baseline( + local_path=str(fixture_dir), + level=3, + output_format="json", + auto_init_config=False, # fixtures ship their own .project/ + attest=False, + prefer_upstream=False, + ) + return AuditResult.from_mcp_json(json.loads(raw)) +``` + +`AuditResult.from_mcp_json` and `AuditResult.from_harness_report` are two small factory functions on the same dataclass -- they normalize both output shapes into a common form the comparator operates on. + +**Rationale**: `audit_openssf_baseline` is the actual MCP tool implementation. The MCP protocol wrapper (`darnit.server.factory`) just JSON-serializes the return value; there's no other transformation. + +**Alternatives considered**: + +- Spawn a subprocess `darnit serve` and call via JSON-RPC: rejected in clarify Q4 -- too slow, no diagnostic benefit for what could regress in the audit layer. +- Call `run_sieve_audit` directly (the shared kernel of both paths): rejected -- would be a "test tests itself" tautology since both the MCP tool and the harness are wrappers around `run_sieve_audit`. We want to catch bugs in the WRAPPERS. + +## R5. Adversarial test seeding (SC-001, SC-003) + +**Decision**: The adversarial tests use a "fake MCP tool result" injection point, not by modifying real audit code. Concretely: + +```python +def test_comparator_catches_pass_to_fail_divergence(): + """SC-001: Deliberately construct a diverging pair and assert the + comparator reports a hard failure.""" + mcp_result = AuditResult(controls=[ + Control(id="X", status="PASS", authority="dispositive"), + ]) + harness_result = AuditResult(controls=[ + Control(id="X", status="FAIL", authority="dispositive"), + ]) + drifts = compare(mcp_result, harness_result) + disallowed = [d for d in drifts if not d.is_allowed_drift] + assert len(disallowed) == 1 + assert disallowed[0].control_id == "X" + +def test_comparator_failure_message_lists_all_drifts(): + """SC-003: N seeded divergences produce N table rows.""" + ... +``` + +The adversarial tests exercise `comparator.compare()` and `format_drift_table()` directly with hand-constructed inputs. Feature 026 and 027 code stays untouched. + +**Rationale**: Adversarial tests should test the COMPARATOR, not simulate a broken darnit. Simulating a broken darnit would require monkey-patching product code, which is more fragile and adds no signal about whether the comparator itself catches real drift. + +**Alternatives considered**: + +- Property-based tests (hypothesis) generating random `AuditResult` pairs: could add later as a follow-up; overkill for the MVP where the drift classes are enumerable. +- Fault-injection at the harness layer: rejected -- reaches into product internals; a fault-injection API is bigger scope than the whole feature. + +## R6. Skill Markdown parsing (Tier 2) + +**Decision**: `skill_markdown_parser.py` uses regex-based extraction with an explicit best-effort contract: + +1. Match the summary counts pattern: `\d+/\d+ pass`, `\d+/\d+ fail`, etc. (skill's current format from PR #365 review notes). +2. Match per-control claims: heading-shaped patterns like `**OSPS-XX-01.01**: PASS` and enumerated status references. +3. If either extraction fails, return a `SkillReport` with `parseable = False` and the raw Markdown attached; Tier 2 fails with a "skill output unparseable" verdict. + +The parser lives in `tests/darnit/parity/tier2/skill_markdown_parser.py`. Its tests use golden files -- captured skill outputs from earlier runs. + +**Rationale**: The skill's output format is not a stable contract; we cannot depend on it. A best-effort parser with a distinguishable "unparseable" failure class is the honest approach. + +**Alternatives considered**: + +- LLM-based summarization of the skill's output: rejected -- introduces another API call, another model whose output we'd need to trust, and defeats the point of a diagnostic test. +- Ask the skill to emit structured JSON: rejected in clarify Q2 (spec's FR-006a) -- a diagnostic feature must not modify the thing it diagnoses. +- HTML/Markdown AST parsing (mistune, markdown-it-py): considered; adds a dep. If regex parsing turns out to be too brittle, we can swap the parser implementation later without changing the SkillReport shape. + +## R7. Claude Agent SDK invocation shape + +**Decision**: `claude_agent_sdk_client.py` invokes the SDK with: + +- Pre-configured system prompt matching what Claude Code loads for the `/darnit-audit` skill (captured verbatim from `.claude/skills/darnit-audit/` if present, or from a snapshot committed to `tests/darnit/parity/tier2/skill_prompt_snapshot.md`). +- Tool allow-list: only the darnit MCP tools this skill invokes (`audit_openssf_baseline`, `list_available_checks`, etc.). No general-purpose tools; no filesystem write. +- Model pinned to whatever the current default is (`anthropic:claude-sonnet-5` per feature 025/026 default). Configurable via env var so we can rerun a check against a specific model in an investigation. +- Deterministic mode where the SDK offers it (temperature=0 or lowest available). +- Turn cap: bounded by an explicit `max_turns` (default 20) so a runaway skill can't burn budget. + +**Rationale**: The whole point of the SDK vs subprocess is scripted, deterministic invocation. Explicit prompt + tool grants + turn cap + temperature is what makes runs reproducible. + +**Alternatives considered**: + +- Freshly-authored prompt (not the skill's real one): rejected -- the test would measure a hypothetical, not the actual skill users experience. +- No turn cap: rejected -- a bug in the skill's prompting could cost real money. + +## R8. Tier 2 CI workflow shape (governance-critical) + +**Decision**: `.github/workflows/parity-tier2.yml`: + +```yaml +on: + workflow_dispatch: + inputs: + fixture_glob: + description: "Fixture directory glob (default: all)" + default: "*" + required: false + +jobs: + tier2: + runs-on: ubuntu-latest + environment: parity-tier2 # <-- gated Environment, required-reviewer list + permissions: + contents: read + steps: + - checkout, setup-python, uv sync --dev + - preflight: log actor + SHA to job summary (FR-007a audit trail) + - run: uv run python tests/darnit/parity/tier2/run.py --fixture-glob "${{ inputs.fixture_glob }}" + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + - upload artifacts: parity-artifacts/ +``` + +Key config beyond the YAML: + +- The `parity-tier2` Environment MUST be configured in GitHub UI with (a) required-reviewer list, (b) `ANTHROPIC_API_KEY` as an Environment secret (NOT a repo secret). +- No other workflow references `secrets.ANTHROPIC_API_KEY` (SC-005a verifiable via grep). +- `permissions: contents: read` only; no write scope so a compromised workflow can't push commits. + +**Rationale**: This is exactly the GitHub-Actions Environment pattern for high-cost or high-risk jobs. Blocks unauthorized dispatch at the platform layer, not in code. + +**Alternatives considered**: + +- Approve every workflow_dispatch invocation via a separate approval step: same effect as Environment reviewer but adds YAML complexity. +- Move the API key to a self-hosted runner: overkill for the current threat model. + +## R9. Fixture authoring cost + +**Decision**: The MVP fixture corpus reuses feature 026's `minimal_llm_repo` (PENDING_LLM category) and adds three new synthetic ones: + +- `all_pass_repo/`: satisfies every Level-1 control the fixture wants to include. Explicit `.project/project.yaml` with all required context values. Small file set (LICENSE, README, SECURITY.md, minimal `.github/`). +- `all_fail_repo/`: bare repo; almost nothing. `.baseline.toml` present but repo files intentionally absent. Every control at every level fails. +- `mixed_repo/`: some controls satisfied, some deliberately not. Explicit `.project/project.yaml` for the "yes" side. About 6 controls PASS, 6 FAIL, 3 WARN. + +Each fixture ships with a `parity.toml` declaring its category + expected counts (from an initial run of the tool during fixture authoring). + +**Rationale**: Four fixtures is enough to cover the four SC-008 categories. Smaller-is-better for CI time. Additional fixtures land in follow-up PRs when specific corner cases surface. + +**Alternatives considered**: + +- Copy real repositories (curl, kubernetes, etc.) as fixtures: rejected -- churn, license concerns, and audit results depend on live GitHub API responses that a fixture can't provide deterministically. +- Generate fixtures programmatically from a fixture-authoring DSL: rejected as premature abstraction; four hand-written fixtures is manageable. + +## R10. Reporting on green runs (FR-013) + +**Decision**: Even on Tier 1 green runs, the comparator emits a summary line per fixture: + +``` +[tier1] all_pass_repo: 3 controls compared, 3 agreed, 0 diverged +[tier1] pending_llm_repo: 6 controls compared, 5 agreed, 1 allowed-drift (PENDING_LLM->WARN) +``` + +Emitted via `pytest.warns`-like sidechannel: a per-fixture line in the pytest report. Not a warning (doesn't imply anything is wrong); an informational report that CI can grep for evidence of a green run's shape. + +Tier 2's report shape is similar, written to the job summary (`GITHUB_STEP_SUMMARY`). + +**Rationale**: FR-013 hard rule -- report on green runs too. This gives the maintainer a check that "the test ran and looked at N controls" rather than the tests being silently no-op. + +**Alternatives considered**: + +- Emit only on failure: rejected -- FR-013 rules that out, and rightly so; a test suite that says nothing on green is one that could silently disable itself. + +## Summary of Phase 0 outcome + +- Every technical unknown for Phase 1 design has a concrete decision above. +- No new production dependencies. The Claude Agent SDK is test-only, added to a workspace dev group. +- SC-006 hard constraint mechanically holds: no `packages/darnit/pyproject.toml` or `packages/darnit-baseline/pyproject.toml` change is planned. +- Governance property (FR-007a + SC-005a) is enforced at the GitHub Environment layer; the plan-phase workflow YAML has the right shape. +- Adversarial-test strategy exercises the comparator directly with hand-built inputs; no product-code fault injection required. +- Skill Markdown parser is best-effort with an "unparseable" failure class distinct from "disagreement." diff --git a/specs/028-audit-parity-tests/spec.md b/specs/028-audit-parity-tests/spec.md new file mode 100644 index 0000000..2551c4f --- /dev/null +++ b/specs/028-audit-parity-tests/spec.md @@ -0,0 +1,144 @@ +# Feature Specification: Two-Tier Audit Parity Tests + +**Feature Branch**: `028-audit-parity-tests` + +**Created**: 2026-08-09 + +**Status**: Draft + +**Input**: User description: "Add a two-tier parity test suite that verifies the darnit audit's output is consistent across three consumers: direct MCP tool call, `darnit harness`, and the `/darnit-audit` coding-agent skill." + +## Clarifications + +### Session 2026-08-09 + +- Q: How does Tier 2 invoke the `/darnit-audit` coding-agent skill? -> A: Claude Agent SDK. Purpose-built for scripted agent invocations; provides deterministic turn/tool-call data; no interactive I/O. Added as a TEST-ONLY dependency (product packages unchanged). Follow-up issue #368 tracks equivalent OpenAI-SDK / other-provider parity checks so this feature isn't provider-locked in the long term; those are separate features scoped to their own SDKs. +- Q: What counts as "the skill's summary" for Tier 2 comparison? -> A: The skill's final assistant message. That is exactly what a human operator reads and believes; a diagnostic feature must compare user-facing output. Parsing is heuristic (Markdown scraping); a parse failure is a distinct failure class ("skill output unparseable") separate from a disagreement ("skill and tool disagree"). Structured-artifact alternatives (asking the skill to emit JSON alongside its Markdown) were rejected as intrusive -- a diagnostic feature should not modify the thing it diagnoses. +- Q: How often does Tier 2 run? -> A: Manual-only for the MVP (`workflow_dispatch`; no schedule). Rationale is governance, not cost: the darnit repo is under neutral governance, and the ANTHROPIC_API_KEY that would run this belongs to a specific company. Automated scheduled runs would charge that company's account for community activity. Manual dispatch preserves accountability -- an authorized maintainer explicitly consents to each run. Access control on the workflow / secret is a hard requirement (see FR-007a). Follow-up issue #369 captures the "add scheduled cadence once a governance-appropriate key-sourcing model exists" scope. +- Q: What Tier 1 calls as "the MCP tool" in its parity comparison. -> A: Direct Python function call: `from darnit_baseline.tools import audit_openssf_baseline; result = audit_openssf_baseline(local_path=..., level=..., output_format="json")`. The MCP protocol layer is a thin serialization wrapper around this function; the audit logic is what could regress. Direct call is faster, deterministic, and requires no server bootstrap. JSON-RPC serialization is out of scope for Tier 1; a separate narrow test can cover it if the need arises. +- Q: Fixture metadata file format. -> A: `parity.toml` at each fixture's root, TOML-parsed. Matches darnit's TOML-first convention (Constitution III). No code execution at load time (security -- rules out a Python-literal metadata file). Uses stdlib `tomllib`, no new dependency. Expected schema (fields documented in the plan phase): `[expected] counts.pass`, `counts.fail`, `counts.warn`, `counts.pending_llm`; `has_pending_llm: bool`; `category: "all_pass" | "all_fail" | "mixed" | "pending_llm"` for SC-008 corpus-inventory verification. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Maintainer catches a harness regression before merging (Priority: P1) + +A maintainer opens a PR that changes the harness's `_collect_unanswered` logic. CI runs the Tier 1 parity test suite on the fixture corpus. The suite invokes the audit via both the direct MCP tool entry point AND via `darnit harness`, then diffs the per-control status. If any control differs beyond the documented allowed drift (harness resolves PENDING_LLM to WARN or an LLM-decided status), CI fails with a human-readable diff table showing exactly which control changed and how. + +**Why this priority**: This is the mechanical safety net that fell out of the PR #365 review. The harness and the MCP tool share a `run_sieve_audit` code path today, but they consume the output differently (harness runs an LLM continuation loop; MCP tool leaves PENDING_LLM in place). A regression that makes the harness silently disagree with the MCP tool would undermine the whole "faithful to the audit" property; this tier catches it in seconds on every PR. + +**Independent Test**: Add or modify the harness code, run `uv run pytest tests/darnit/parity/tier1/ -q`, and observe the pass/fail. The test suite requires no live API calls (uses `MockLLMStep`); it produces a diff table on failure that identifies exactly which control-level statuses differ. + +**Acceptance Scenarios**: + +1. **Given** the harness and MCP tool agree on every control's status for a fixture, **When** the Tier 1 test runs, **Then** it passes. +2. **Given** a change to the harness makes it report FAIL for a control the MCP tool reports PASS, **When** the Tier 1 test runs, **Then** it fails with an assertion message that names the control and both statuses. +3. **Given** the MCP tool leaves a control PENDING_LLM and the harness resolves it to WARN via the LLM continuation loop, **When** the Tier 1 test runs, **Then** it PASSES (this is the sole documented allowed drift). +4. **Given** a fixture is added to the corpus but no test task is added, **When** the Tier 1 test runs, **Then** the new fixture is automatically covered (the test suite iterates the fixture directory). + +--- + +### User Story 2 - Maintainer discovers coding-agent skill drift (Priority: P2) + +A nightly (or weekly) CI job runs the Tier 2 parity check. For each fixture in the corpus, it captures the raw MCP tool JSON output, then invokes the `/darnit-audit` coding-agent skill on the same fixture via the Claude Agent SDK. It parses the skill's Markdown summary for PASS/FAIL/WARN counts and per-control claims, and diffs them against the raw tool output. If the skill's summary reclassifies any control's status differently than the raw output, CI captures both artifacts (the skill's Markdown + the tool's JSON) for human inspection. + +**Why this priority**: The reason issue #366 exists. The `/darnit-audit` skill was observed reclassifying WARN -> PASS in its summary; a maintainer would like to know when this happens without discovering it during manual testing. Priority 2 rather than 1 because the check requires a live API call (rate limits, cost), so it can't run on every PR. + +**Independent Test**: Manually trigger the Tier 2 job (or wait for the scheduled run). Verify (a) it exits 0 when the skill's summary agrees with the raw tool output on every control's status, (b) it exits non-zero when they disagree, and (c) the failure artifact contains both the skill Markdown and the tool JSON side-by-side. + +**Acceptance Scenarios**: + +1. **Given** the skill's summary reports the same PASS/FAIL/WARN counts as the raw MCP tool for every fixture, **When** the Tier 2 job runs, **Then** it exits successfully. +2. **Given** the skill silently reclassifies a WARN control as PASS in its summary, **When** the Tier 2 job runs, **Then** it fails with a per-control diff and the raw artifacts attached. +3. **Given** a control's Check step had `dispositive` authority evidence and the skill's summary matches the tool's status for it, **When** the Tier 2 job runs, **Then** that control passes the check. +4. **Given** a control's Check step had `suggestive` authority evidence and the skill's summary changed its status, **When** the Tier 2 job runs, **Then** the check STILL fails -- the skill has no license to reinterpret verdicts regardless of authority level. Any status change from the skill layer relative to the raw tool output is a hard failure. + +--- + +### User Story 3 - Fixture author adds coverage for a new audit corner (Priority: P3) + +A maintainer notices that no fixture exercises the case where every control PASSes cleanly. They add a new fixture repo under `tests/darnit/parity/fixtures/`, populate it with the file structure that satisfies all Level-1 controls, and run the parity tests. Both tiers automatically discover the new fixture and run against it. No test-task edit is required. + +**Why this priority**: Auto-discovery of fixtures is convenience, not a load-bearing property. The parity suite works with a single fixture; more fixtures produce broader coverage. Priority 3 because the discovery mechanism is nice-to-have, not required. + +**Independent Test**: Add a directory `tests/darnit/parity/fixtures/all_pass_repo/` containing the requisite files; run `uv run pytest tests/darnit/parity/ -q` (Tier 1); verify the new fixture's tests are collected and pass. + +**Acceptance Scenarios**: + +1. **Given** the fixtures directory contains N fixtures, **When** the Tier 1 suite runs, **Then** each fixture is exercised at least once (verified by test collection count). +2. **Given** a new fixture is added by placing a directory under the fixtures root, **When** pytest is next invoked, **Then** the new fixture is automatically included without any test file changes. + +--- + +### Edge Cases + +- **Fixture with no controls loaded** (empty `.baseline.toml`): Tier 1 should not silently pass; the harness reports SETUP_ERROR (exit 2) for this case, and the MCP tool reports zero controls. The suite treats "both reported zero controls" as a valid parity outcome for a fixture explicitly labeled empty, and a mismatch (one path errors, the other returns zero) as a failure. +- **Fixture that produces PENDING_LLM in the MCP tool path**: Tier 1 records the PENDING_LLM verdict from the MCP tool and the resolved verdict (WARN or a specific LLM-decided status) from the harness. This ONE class of drift is explicitly allowed and documented; no other drift is. +- **Skill returns no Markdown summary** (e.g., skill invocation times out): Tier 2 reports a distinct failure class ("skill did not produce a summary") separate from "skill and tool disagree." The failure artifact still captures whatever the skill produced (if anything). +- **Skill's Markdown parseable but ambiguous** (skill reports "56/66 pass" but doesn't enumerate per-control): Tier 2 falls back to a summary-count comparison ("skill's PASS count vs tool's PASS count"). If those differ, fail; if they match, warn that per-control comparison was not possible for this run. +- **ANTHROPIC_API_KEY absent for Tier 2**: fail with a clear setup error ("Tier 2 requires ANTHROPIC_API_KEY; skipping or failing per configuration"). Never silently pass. +- **New allowed drift discovered**: any change to the "PENDING_LLM -> resolved" allowance must be a deliberate spec change. Adding a second allowed drift class requires updating this spec and the test's allowed-drift table. +- **Fixture removal**: if a fixture is deleted, the auto-discovery mechanism produces fewer tests but no failure. A test collection count check catches accidental fixture deletion during unrelated refactors (informational only). +- **Tier 2 rate limit hit mid-run**: capture the partial results; fail with a clear "rate limited" message; do NOT retry automatically. A subsequent manual dispatch picks up from a clean state. +- **Unauthorized dispatch attempt**: a GitHub user without the reviewer role tries to trigger the Tier 2 workflow. GitHub Actions' Environment gate blocks the run; the workflow never executes; no API budget is consumed. This is by design (FR-007a); no additional darnit-side logic is required beyond configuring the Environment correctly. +- **Missing reviewer approval**: a dispatched run waits at the approval gate indefinitely. Approval timeout is a GitHub configuration setting (spec does not fix it; a plan-phase choice). + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST provide a Tier 1 automated test that, for every fixture in the corpus, invokes the audit via BOTH the direct MCP tool entry point (Python function call: `audit_openssf_baseline(local_path=..., level=..., output_format="json")`) AND `darnit harness` in-process (no CLI subprocess, no MCP server bootstrap), and compares the per-control status output of both paths. +- **FR-002**: Tier 1 MUST treat exactly one class of drift as allowed: a control that the direct MCP tool reports as PENDING_LLM is allowed to be reported as any non-PENDING_LLM status by the harness (WARN, PASS, FAIL, N/A, or ERROR, depending on what the LLM continuation loop resolved it to). Every other per-control status difference is a hard failure. +- **FR-003**: Tier 1 MUST NOT require a live LLM API call. The harness's LLM step MUST be pluggable and the test MUST inject a `MockLLMStep` or equivalent so the whole Tier 1 suite runs offline. +- **FR-004**: On Tier 1 failure, the assertion message MUST include a human-readable table listing every diverging control with columns for control_id, MCP tool status, and harness status. The table MUST be readable in a terminal (fixed-width, no ANSI escapes required). +- **FR-005**: Tier 1 MUST run in under 60 seconds for the full fixture corpus (defined below). Individual fixture tests SHOULD complete in under 10 seconds each. +- **FR-006**: The system MUST provide a Tier 2 automated check that, for every fixture in the corpus, (a) invokes the direct MCP tool and captures its JSON output, (b) invokes the `/darnit-audit` coding-agent skill via the Claude Agent SDK on the same fixture and captures the SKILL'S FINAL ASSISTANT MESSAGE (the artifact a human user actually reads), (c) diffs the skill's per-control claims and PASS/FAIL/WARN counts against the tool's raw output. The Claude Agent SDK is a TEST-ONLY dependency; product packages MUST NOT gain a runtime dep. Parallel parity-check features for other provider SDKs (OpenAI, etc.) are out of scope and tracked as separate follow-up issues. +- **FR-006a**: Tier 2 MUST NOT modify the `/darnit-audit` skill (its prompt, its tool grants, its output format) as a precondition of the test. The skill is what it is; the parity check parses whatever the skill produces. If the skill's final-message format changes such that the parser can no longer extract per-control claims, Tier 2's failure surfaces as "skill output unparseable" -- a distinct failure class from "skill and tool disagree" -- so a maintainer can distinguish a broken parser from a real drift. +- **FR-007**: Tier 2 MUST run only under manual dispatch (`workflow_dispatch` in GitHub Actions) for the MVP. No `schedule:` trigger. Rationale is governance: the darnit repo is under neutral governance, and the ANTHROPIC_API_KEY that runs this belongs to a specific company that MUST NOT be charged for unattended community-triggered activity. A follow-up issue captures the "add scheduled cadence" scope once a governance-appropriate key-sourcing model exists (options include: dedicated community-owned API key with usage cap, per-maintainer BYO-key model, GitHub-Environment-approval-gated runs with a shared key). +- **FR-007a**: Access control on Tier 2 workflow invocation MUST prevent an unauthorized party from triggering a run that charges the API-key-owner's account. Concretely: (a) the workflow lives in a GitHub Actions Environment configured with a required-reviewer list (only listed reviewers can approve a dispatch); (b) the `ANTHROPIC_API_KEY` secret lives in that Environment, not at the repository level, so it is not exposed to any workflow outside the gated environment; (c) the workflow definition includes a preflight check that logs the actor and the SHA before spending any API budget, so a post-hoc audit can attribute cost. A misconfigured deployment where the API key is exposed to workflows outside the reviewer gate is treated as a severity-1 governance bug. +- **FR-007b**: Tier 2 MUST NOT accept an operator-provided API key as a workflow input for the MVP. Reason: an input-key model bypasses the Environment/reviewer gate and re-opens the "arbitrary community member spends someone else's money" hole in a subtler form. A future BYO-key model may reintroduce this; the follow-up issue tracks that decision. +- **FR-008**: Tier 2 MUST fail on any per-control status difference between the skill's summary and the raw tool output, regardless of the control's authority level. The skill has no license to reinterpret verdicts. +- **FR-009**: On Tier 2 failure, the CI job MUST attach both the skill's raw Markdown output AND the raw MCP tool JSON for the failing fixture as an inspectable artifact, so a maintainer can review the drift without re-running the check. +- **FR-010**: When ANTHROPIC_API_KEY is absent, Tier 2 MUST fail fast with a clear setup error. Silent-skip is forbidden -- an operator MUST know Tier 2 did not run. +- **FR-011**: The fixture corpus MUST include at least four fixtures covering distinct audit-output shapes: (a) all-PASS, (b) all-FAIL, (c) mixed PASS/FAIL/WARN, (d) at least one control that produces PENDING_LLM under the MCP tool path. The existing `minimal_llm_repo` fixture from feature 026 counts toward (d). +- **FR-012**: Fixtures MUST be auto-discovered: any directory under a documented fixtures root is treated as a fixture without requiring a corresponding test file edit. Adding or removing a fixture only requires changing files in that directory. +- **FR-012a**: Each fixture MAY carry a `parity.toml` file at its root declaring the expected shape of its output. TOML-parsed via stdlib `tomllib`; no code execution. Schema (documented in plan phase) at minimum includes `[expected] counts.pass`, `counts.fail`, `counts.warn`, `counts.pending_llm`, and `category` (one of `"all_pass"`, `"all_fail"`, `"mixed"`, `"pending_llm"`). Fixtures without a `parity.toml` are treated as "shape unspecified" -- parity across paths is still asserted, but corpus-inventory (SC-008) checks skip them. +- **FR-013**: Both tiers MUST produce a report that includes the number of controls checked, the number that agreed across paths, and the number that diverged (with drift classification for Tier 1). The report is emitted regardless of pass/fail so counting evidence is captured even on green runs. +- **FR-014**: The parity test suite MUST NOT modify the darnit product code as a side effect. The tests are pure consumers of the existing MCP tool, harness, and skill surfaces. Any test-side helper that requires a product change (e.g., an injection seam that doesn't exist) is spec-scope creep and MUST be flagged as a follow-up rather than silently added. +- **FR-015**: Tier 1 test failures MUST be reproducible from a git commit hash + fixture directory alone. No hidden global state, no time-of-day sensitivity, no ordering dependency across fixtures. Deterministic execution is a hard requirement. +- **FR-016**: The parity test suite MUST close issue #366 when merged. Its purpose is diagnosis (finding drift) not remediation (fixing the skill). Any drift Tier 2 discovers is filed as a separate issue. + +### Key Entities + +- **Fixture**: A directory containing everything needed to run an audit -- a `.baseline.toml`, a `.project/project.yaml` (optional), and whatever repo files the controls reference. Each fixture has a stable identifier (its directory name) and an optional `parity.toml` metadata file at its root declaring the expected shape of its output (TOML-parsed; schema per FR-012a). Fixtures without `parity.toml` still participate in inter-path parity assertions but are skipped for corpus-inventory checks. +- **AuditResult**: The unified representation of one audit's output. Contains the list of controls with their statuses, authority levels, and any pending-LLM markers. Both the MCP tool path and the harness path produce this shape; the parity test compares them. +- **DriftEntry**: One row of the diff table produced on Tier 1 failure. Fields: fixture identifier, control_id, path-A status, path-B status, whether the drift is in the documented allowed set. +- **SkillReport**: The Tier 2-parsed view of the coding-agent skill's Markdown output. Contains the counts (PASS/FAIL/WARN) the skill reported and per-control claims (extracted from the Markdown structure). Not part of the darnit product; a test-only intermediate. +- **ParityReport**: The end-of-run summary emitted by either tier. Contains per-fixture pass/fail, drift counts, and (for Tier 2 failure) the raw artifact paths. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Tier 1 catches 100% of harness-vs-tool per-control divergences in the fixture corpus other than the documented PENDING_LLM-resolution drift. Verified by an adversarial test that deliberately introduces a divergence and asserts Tier 1 fails. +- **SC-002**: The full Tier 1 suite runs in under 60 seconds on a standard developer laptop. Verified by a timing assertion in CI. +- **SC-003**: Every diverging control in a Tier 1 failure has a corresponding row in the failure message. Verified by an adversarial test that seeds N divergences and asserts the failure message contains N rows. +- **SC-004**: Tier 2 catches 100% of skill-vs-tool per-control status divergences in the fixture corpus, regardless of authority level. Verified by an adversarial fixture where the skill (via a mocked SDK response) reclassifies a WARN control as PASS; Tier 2 fails. +- **SC-005**: Tier 2 artifacts on failure include BOTH the skill's raw Markdown AND the tool's JSON for every failing fixture. Verified by inspecting a scripted failure run's artifact directory. +- **SC-005a**: The `ANTHROPIC_API_KEY` secret used by Tier 2 MUST NOT be reachable by any GitHub Actions workflow other than the gated Tier 2 workflow. Verified by grepping `.github/workflows/` for other `secrets.ANTHROPIC_API_KEY` references (result MUST be empty except in the Tier 2 workflow) AND by confirming the secret's storage location is a GitHub Environment (not a repo-level secret) whose only entry point is the reviewer-gated deployment. +- **SC-006**: The parity test suite does not add ANY runtime dependencies to the darnit product packages. Verified by comparing the pre- and post-feature `pyproject.toml` dependency lists in `packages/darnit/` and `packages/darnit-baseline/`. +- **SC-007**: A new fixture added by creating a directory under the fixtures root is exercised by BOTH tiers on the next run, with no test file changes. Verified by adding a fixture in the test suite itself and asserting collection count increases by one. +- **SC-008**: The fixture corpus produces at least one control in each of the four categories: PASS-only, FAIL-only, mixed, and PENDING_LLM. Verified by a corpus-inventory test that counts the categories represented. +- **SC-009**: An issue #366 status check (via `gh issue view 366`) MUST show "Closed" within one working day of this feature's PR merging. Manual verification. + +## Assumptions + +- The parity tests live under `tests/darnit/parity/` as a new pytest package. Tier 1 tests live in `tests/darnit/parity/tier1/`; Tier 2 tests + scaffolding live in `tests/darnit/parity/tier2/`. Fixture directories live under `tests/darnit/parity/fixtures/`. +- The Tier 2 job is a GitHub Actions workflow with a `schedule:` trigger and appropriate secret injection for `ANTHROPIC_API_KEY`. The exact YAML shape is a plan-phase concern. +- The `/darnit-audit` skill invocation from Tier 2 goes through the Claude Agent SDK (as opposed to Claude Code CLI). The SDK provides deterministic invocation: no interactive turns, no user prompting, prompts and tool grants pre-configured. The SDK is a runtime dependency of the TEST suite only, never the product. +- Parsing the skill's Markdown summary is a lossy operation; the skill's output format is not a stable contract. The test suite's Markdown parser lives in `tests/darnit/parity/tier2/skill_markdown_parser.py` and produces a best-effort extraction. If the skill changes its output format such that the parser breaks, Tier 2's failure mode is "could not parse skill output" -- distinct from "skill and tool disagree." +- Feature 026 (`darnit harness`) is a hard dependency; the harness code must exist before Tier 1 can be written. Feature 027 (interactive resolvers) is NOT a dependency -- the parity tests do not exercise interactive answer collection. +- The Claude Agent SDK's dependency, install path, and version pinning are plan-phase decisions. The spec assumes a reputable, published SDK exists; if it does not, Tier 2's implementation approach may change (e.g., invoke `claude` CLI as a subprocess). +- Fixture repos are lightweight; they contain only the files necessary for the controls they exercise. No large binary blobs. Fixtures should git-clone in under 5 seconds and audit in under 10 seconds each. +- Tier 1's "under 60 seconds" budget accommodates a fixture corpus of 4-6 fixtures. If the corpus grows past 20 fixtures, the budget may need revisiting; that is a spec change. +- The `/darnit-audit` skill's own version is captured in Tier 2 artifacts (from the SDK's provenance response, if available; otherwise from the invocation config) so a maintainer can correlate a drift with a specific skill version. +- The tests are diagnostic. They do NOT propose fixes for any drift they discover. A Tier 2 failure filed as an issue triggers a separate feature to decide the fix. diff --git a/specs/028-audit-parity-tests/tasks.md b/specs/028-audit-parity-tests/tasks.md new file mode 100644 index 0000000..c69143a --- /dev/null +++ b/specs/028-audit-parity-tests/tasks.md @@ -0,0 +1,345 @@ +--- +description: "Tasks for feature 028: Two-Tier Audit Parity Tests -- MCP tool / harness / coding-agent skill diagnosis" +--- + +# Tasks: Two-Tier Audit Parity Tests + +**Input**: Design documents from `specs/028-audit-parity-tests/` + +**Prerequisites**: plan.md (loaded), spec.md (loaded, 5 clarifications), research.md (loaded, 10 decisions), data-model.md (loaded), contracts/{tier1-parity-invariant,tier2-workflow,parity-toml-schema}.md (loaded), quickstart.md (loaded). + +**Tests**: Test tasks included. This entire feature IS a test suite; every FR maps to a concrete pytest module or a workflow-config assertion. Load-bearing SCs: SC-001/003 (adversarial-drift detection), SC-002 (Tier 1 <60s), SC-004 (Tier 2 catches skill reclassification), SC-005a (no ANTHROPIC_API_KEY exposure outside gated workflow), SC-006 (zero product deps added), SC-008 (four fixture categories). + +**Organization**: Tasks grouped by user story per spec.md. Feature 026 (harness) is a hard dependency. Feature 027 (interactive resolvers) is NOT a dependency; parity tests do not exercise interactive answer collection. + +**Branch base**: `026-harness-with-stage1` (PR #365, still open). Rebase to `main` once #365 lands. Do NOT branch from `main` directly -- feature 028 depends on 026's `HarnessRun` and `MockLLMStep`. + +## Format: `[ID] [P?] [Story?] Description` + +- **[P]**: Parallelizable with other [P] tasks in the same phase +- **[Story]**: Which user story (US1, US2, US3) +- File paths are exact and repository-relative +- Closes #366 on merge + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Create the parity-test tree. No product-package changes. The Claude Agent SDK dep goes into a test-only dev group (SC-006). + +- [X] T001 Create `tests/darnit/parity/` directory with an empty `__init__.py`. Add `tests/darnit/parity/tier1/__init__.py` and `tests/darnit/parity/tier2/__init__.py`. Add `tests/darnit/parity/fixtures/` (empty container; fixtures added per-user-story). + +- [X] T002 [P] Add `claude-agent-sdk` to the workspace-level dev-group `pyproject.toml` at repo root (or the appropriate `uv`-workspace dev-group location; see `packages/darnit/pyproject.toml` for the pattern). MUST NOT modify `packages/darnit/pyproject.toml` or `packages/darnit-baseline/pyproject.toml` -- SC-006 requires zero product dep changes. Include a comment identifying it as a Tier 2 test-only dep. + +- [X] T003 [P] Add a top-level `.gitignore` entry for `parity-artifacts/` if not already ignored, so Tier 2's local runs don't accidentally commit skill-invocation transcripts. + +**Checkpoint**: Directory scaffolding exists; `uv sync --dev` installs `claude-agent-sdk` for maintainers; product packages are untouched. + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: The shared data types + comparator logic + `parity.toml` parser that BOTH Tier 1 and Tier 2 depend on. Each task creates a self-contained module with its own tests. + +**CRITICAL**: No user-story tasks can proceed until this phase is complete. + +- [X] T004 Create `tests/darnit/parity/tier1/comparator.py` per data-model.md sections 3-5: + - `Control` (frozen dataclass: `id`, `status` Literal, `authority` Literal | None, `level` int | None) + - `AuditResult` (frozen dataclass with `controls: tuple[Control, ...]`, `source: Literal["mcp_tool", "harness"]`) + - `AuditResult.from_mcp_json(payload: dict) -> AuditResult` classmethod parsing the shape returned by `audit_openssf_baseline(output_format="json")` + - `AuditResult.from_harness_report(report: HarnessReport) -> AuditResult` classmethod + - `DriftEntry` (frozen dataclass with `fixture_name`, `control_id`, `mcp_status`, `harness_status`; `is_allowed_drift` property per T1-2 canonical table in `tier1-parity-invariant.md`) + - `ParityReport` (frozen dataclass; `disallowed_drifts` property, `is_green` property, `format_summary_line()`, `format_failure_table()` per FR-004 fixed-width Markdown; no ANSI) + - `compare(mcp: AuditResult, harness: AuditResult, fixture_name: str) -> ParityReport` function implementing the T1-8 allowed-drift table exactly + +- [X] T005 [P] Create `tests/darnit/parity/tier1/fixture_meta.py` per contract `parity-toml-schema.md`: + - `ParityMetadata` dataclass with fields matching the schema (`category`, `has_pending_llm`, `strict`, `counts`, `controls`) + - `load_parity_metadata(fixture_dir: Path) -> ParityMetadata | None` -- returns None if `parity.toml` absent (PT-2 optional), raises `ValueError` on malformed TOML (PT-4), warns on unknown keys (PT-9), validates `category` literal + count non-negativity + `has_pending_llm` vs `counts.pending_llm > 0` agreement (PT-5, PT-6, PT-8) + - Uses stdlib `tomllib.load(...)`; no new dep (PT-3) + +- [X] T006 [P] Create `tests/darnit/parity/tier1/test_comparator.py`: + - Enumerate every (mcp_status, harness_status) pair from the six possible statuses (36 pairs total) and assert `compare()` classifies each per the T1-8 table (T1-9 mechanical enumeration). + - Test `AuditResult.from_mcp_json` and `from_harness_report` produce equivalent shapes from equivalent inputs. + - Test `ParityReport.format_failure_table()` output is fixed-width Markdown (no ANSI escapes; verifiable via `assert "\033" not in output`). + - Test `ParityReport.format_summary_line()` matches the FR-013 evidence-line shape. + - **Determinism (MC4 fix, FR-15)**: run `compare()` twice with identical inputs and assert byte-identical `format_failure_table()` output AND byte-identical `format_summary_line()` output. Catches dict-iteration-order or time-dependent regressions. + +- [X] T007 [P] Create `tests/darnit/parity/tier1/test_fixture_meta.py`: + - PT-3: parses a valid `parity.toml` via `tomllib`. + - PT-4: malformed TOML raises with a clear message. + - PT-5: unknown `category` value fails validation. + - PT-6: `has_pending_llm=true` with `counts.pending_llm=0` fails. + - PT-8: negative count fails. + - PT-9: unknown key produces a warning but does not fail. + - PT-2: `load_parity_metadata(fixture_with_no_parity_toml)` returns None (not an error). + +**Checkpoint**: `uv run pytest tests/darnit/parity/tier1/test_comparator.py tests/darnit/parity/tier1/test_fixture_meta.py -q` passes. Comparator + metadata parser are locked; user-story tests can consume them. + +--- + +## Phase 3: User Story 1 -- Tier 1 MCP-vs-harness parity (P1) 🎯 MVP + +**Goal**: Every PR that touches the harness or MCP tool triggers a Tier 1 parity check across the fixture corpus. A regression that makes the harness silently disagree with the MCP tool fails CI within 60 seconds with a human-readable diff table. + +**Independent Test**: With the fixture corpus in place, `uv run pytest tests/darnit/parity/tier1/ -q` runs and passes; a scripted regression (adversarial test with a hand-built diverging AuditResult pair) fails the comparator with the expected table. + +### Fixtures for US1 + +- [X] T008 [P] [US1] Create `tests/darnit/parity/fixtures/all_pass_repo/`: + - `.baseline.toml` selecting a small subset of Level-1 controls this fixture is designed to satisfy (LICENSE presence, SECURITY.md presence, etc. -- pick 4-6 dispositive controls with no LLM step). + - Repo files satisfying every selected control (LICENSE, SECURITY.md, README, `.github/workflows/ci.yml` if referenced). + - `.project/project.yaml` with any required context values (security_contact etc.) so nothing is PENDING_LLM. + - `parity.toml` with `[expected] category="all_pass" has_pending_llm=false` and matching counts. + - Verify by running `audit_openssf_baseline(local_path=..., level=1, output_format="json")` and confirming every result is `PASS` before committing. + +- [X] T009 [P] [US1] Create `tests/darnit/parity/fixtures/all_fail_repo/`: + - `.baseline.toml` selecting 4-6 dispositive controls (same shape as T008). + - Deliberately absent repo files -- no LICENSE, no SECURITY.md, no relevant `.github/` -- so every selected control FAILs at its `file_exists` step. + - `parity.toml` with `[expected] category="all_fail" has_pending_llm=false` and counts. + +- [X] T010 [P] [US1] Create `tests/darnit/parity/fixtures/mixed_repo/`: + - Roughly 6 PASS, 4 FAIL, 2 WARN across the selected controls. + - `.baseline.toml`, `.project/project.yaml` with partial context (some keys present, some missing so their controls WARN). + - `parity.toml` with `[expected] category="mixed"` and detailed counts. + - Include per-control `[[expected.controls]]` entries for at least three controls the test should watch closely. + +- [X] T011 [P] [US1] Create `tests/darnit/parity/fixtures/pending_llm_repo/`: + - Copy or reuse the shape of feature 026's `minimal_llm_repo` (which includes `STAGE1-REF-SECURITY-01`, a control with an `llm_extract` step that produces PENDING_LLM under the MCP tool). + - `.baseline.toml` selecting `STAGE1-REF-SECURITY-01` plus a few dispositive controls. + - `parity.toml` with `[expected] category="pending_llm" has_pending_llm=true` and counts.pending_llm >= 1. + +### Test infrastructure + tests for US1 + +- [X] T012 [US1] Create `tests/darnit/parity/tier1/conftest.py` implementing fixture auto-discovery per research.md R1: + - `pytest_generate_tests` hook that parametrizes `fixture_dir: Path` from directories directly under `tests/darnit/parity/fixtures/` containing `.baseline.toml`. Test IDs are the fixture directory names. + - **Git-init prerequisite (HC1 fix)**: Both audit paths require the target directory to be a git repository (`prepare_audit` -> `detect_repo_from_git`). Provide a `prepared_fixture(fixture_dir, tmp_path)` fixture that: + - Copies `fixture_dir` recursively into `tmp_path / fixture_dir.name` via `shutil.copytree`. + - Runs `git init --initial-branch=main -q` in the copy (via `subprocess.run(check=True, capture_output=True)`). + - Runs `git -c user.name=test -c user.email=test@example.com commit --allow-empty -q -m init` to create an initial commit. + - Runs `git remote add origin https://github.com/fake-owner/fake-repo.git` so `detect_repo_from_git` yields deterministic owner/repo values. + - Yields the copied directory path. + Mirrors feature 026's `tests/darnit/harness/conftest.py::minimal_llm_repo_tree` fixture pattern verbatim. + - Provides `mcp_tool_result(prepared_fixture)` -- invokes `audit_openssf_baseline(local_path=str(prepared_fixture), level=3, output_format="json", auto_init_config=False, attest=False, prefer_upstream=False)` and parses JSON into an `AuditResult`. + - Provides `harness_result(prepared_fixture)` -- constructs `HarnessRun(local_path=str(prepared_fixture), level=3, llm_step=MockLLMStep(LLMJudgment(outcome="inconclusive", confidence=0.0, reasoning="tier1-mock")), per_call_timeout_s=5, total_run_timeout_s=30)` per research.md R3, awaits `run.run()` via `asyncio.new_event_loop().run_until_complete`, returns `AuditResult.from_harness_report(report)`. + - Env-var isolation autouse: sets `ANTHROPIC_API_KEY="test-key-not-real"` so the harness's credential check passes; tests exercising missing-key paths monkeypatch.delenv explicitly. + +- [X] T013 [US1] Create `tests/darnit/parity/tier1/test_mcp_vs_harness.py`: + - `test_parity(fixture_dir, mcp_tool_result, harness_result, capsys)`: computes `ParityReport = compare(mcp_tool_result, harness_result, fixture_name=fixture_dir.name)`; emits `report.format_summary_line()` via `print` (captured by pytest -s); asserts `report.is_green` with `report.format_failure_table()` as the assertion message (FR-004). + - **FR-013 evidence assertion (MC2 fix)**: use `capsys.readouterr()` (or `caplog` if the impl routes through logging) to capture the summary line; assert the line matches the pattern `re.compile(r"^\[tier1\] " + fixture_dir.name + r": \d+ controls compared, \d+ agreed, \d+ diverged")`. Emitted on EVERY run (green or red), so a silent no-op is caught here. + - Full suite MUST run under 60s total; add a `pytest.mark.timeout(60)` at module level as a safety net. + +- [X] T014 [P] [US1] Create `tests/darnit/parity/tier1/test_comparator_adversarial.py` per research.md R5: + - SC-001: `test_comparator_catches_pass_to_fail_divergence` -- hand-built AuditResult pair with a PASS vs FAIL on the same control_id; assert `compare()` returns a `ParityReport` with `is_green=False` and exactly one disallowed drift. + - SC-003: `test_failure_message_lists_all_drifts` -- seed 5 divergences on 5 different control_ids; assert `format_failure_table()` output contains 5 table rows (count `|` line-starts). + - Allowed-drift positive cases (LC1 fix -- all three): (a) PENDING_LLM (MCP) -> WARN (harness); (b) PENDING_LLM (MCP) -> PASS (harness); (c) PENDING_LLM (MCP) -> FAIL (harness). For each, assert `is_green=True` and `drift.is_allowed_drift=True`. Documents that the T1-8 table's wildcard resolution really is any non-PENDING_LLM. + - Disallowed-drift negative case: PENDING_LLM (harness) -> WARN (MCP); assert `is_green=False`. + - "Missing control" case (T1-3): control appears on one side but not the other; assert this is treated as a hard failure with a "missing control" note. + +- [X] T015 [P] [US1] Create `tests/darnit/parity/tier1/test_corpus_inventory.py`: + - SC-008: iterates fixtures via `load_parity_metadata`, counts fixtures per `category`, asserts every category (`"all_pass"`, `"all_fail"`, `"mixed"`, `"pending_llm"`) has at least one representative. + - Test collection count sanity: at least 4 fixtures are discovered (guards against accidental fixture deletion during unrelated refactors). + +**Checkpoint**: `uv run pytest tests/darnit/parity/tier1/ -q` passes in under 60 seconds. Tier 1 is independently shippable if we stop here. Issue #366's mechanical part is closed by US1. + +--- + +## Phase 4: User Story 2 -- Tier 2 skill drift detection (P2) + +**Goal**: An authorized maintainer dispatches the Tier 2 workflow; it invokes the `/darnit-audit` coding-agent skill via the Claude Agent SDK on each fixture, diffs the skill's final assistant message against the raw MCP tool JSON, and either PASSes or FAILs with a per-control diff report as an artifact. Access to the API key is gated behind a required-reviewer approval on the GitHub Environment. + +**Independent Test**: `uv run python tests/darnit/parity/tier2/run.py --fixture-glob "*" --dry-run` (a mode that stubs the SDK client with a canned response) runs to completion; the artifact directory contains the expected per-fixture files. Then, once merged, an authorized maintainer runs the workflow via `workflow_dispatch` and sees green. + +### Tier 2 machinery (Python) + +- [X] T016 [P] [US2] Create `tests/darnit/parity/tier2/skill_markdown_parser.py` per research.md R6 and data-model.md section 6: + - `SkillReport` frozen dataclass with `parseable`, `raw_markdown`, `counts`, `controls`, `parse_notes` fields. + - `SkillReport.parse(markdown: str) -> SkillReport` best-effort regex parser: + - Extract summary counts (`\d+/\d+ pass|fail|warn` patterns). + - Extract per-control claims (heading-shaped `**OSPS-XX-...**` + explicit status references). + - Return `parseable=True` when both extractions succeed; `parseable=False` otherwise. + - Never raises; sets `parseable=False` on any exception. + +- [X] T017 [P] [US2] Create `tests/darnit/parity/tier2/test_skill_markdown_parser.py`: + - Golden-file tests against captured skill outputs (commit at least three: `golden_all_pass.md`, `golden_mixed_drift.md`, `golden_unparseable.md`). + - `parseable=False` for the unparseable case; `SkillReport.raw_markdown` preserved. + - Redaction sanity: parsed output does NOT contain any credential-shaped substring (regression guard). + +- [X] T018 [P] [US2] Create `tests/darnit/parity/tier2/claude_agent_sdk_client.py` per research.md R7: + - Thin wrapper: `invoke_skill(fixture_dir, model, max_turns) -> str` returning the final assistant message. + - Loads system prompt from `tests/darnit/parity/tier2/skill_prompt_snapshot.md` (created in T023). + - Configures tool allow-list to only the darnit MCP tools referenced by the skill (`audit_openssf_baseline`, `list_available_checks`, `confirm_project_data`). + - `temperature=0` or lowest available; `model` defaults to `anthropic:claude-sonnet-5`; explicit `max_turns` cap (default 20). + - Reads `ANTHROPIC_API_KEY` from env; raises `SetupError` if absent (per FR-010). + +- [X] T019 [P] [US2] Create `tests/darnit/parity/tier2/artifact_writer.py`: + - `write_fixture_artifacts(artifact_dir, fixture_name, mcp_json, skill_markdown, diff_md, metadata)` per data-model.md section 7. + - Creates `parity-artifacts//` with `mcp_tool_result.json`, `skill_final_message.md`, `diff_report.md`, `metadata.json`. + - `metadata.json` includes timestamp, actor (`$GITHUB_ACTOR` if set), SHA (`$GITHUB_SHA` if set), model ID, turn count. + +- [X] T020 [US2] Create `tests/darnit/parity/tier2/diff.py`: + - `diff(mcp_result: AuditResult, skill_report: SkillReport) -> Tier2DiffReport`. + - Per FR-008 + T2-13: any per-control status disagreement is a hard fail regardless of authority. + - Distinguish outcomes: `SUCCESS` (agree), `SKILL_UNPARSEABLE` (skill_report.parseable=False), `COUNTS_DISAGREE` (parseable but summary counts differ from raw), `PER_CONTROL_DISAGREE` (parseable but a control's status differs). + - Generates a Markdown `diff_report.md` for artifact-writer to persist. + +- [X] T021 [US2] Create `tests/darnit/parity/tier2/run.py` (entrypoint invoked from workflow_dispatch): + - CLI: `--fixture-glob ` (default `"*"`), `--dry-run` (stubs SDK client with a canned response for T016/T017 verification without live API). + - For each matching fixture: capture MCP tool JSON via direct Python call; invoke SDK client; parse skill output; run `diff()`; write artifacts. + - Aggregate exit codes per contract T2-13: 0 success, 1 disagreement, 2 unparseable, 3 setup, 4 rate-limit. + - Emit summary line to `GITHUB_STEP_SUMMARY` if the env var is set (T2-14): `"Tier 2 parity check: N fixtures checked, X drifts, Y unparseable, Z rate-limited"`. + - Preflight audit log per T2-7/T2-8: log actor + SHA + fixture-glob to summary BEFORE consuming `ANTHROPIC_API_KEY`. + +### Skill prompt snapshot + +- [X] T022 [P] [US2] Snapshot the current `/darnit-audit` skill's system prompt into `tests/darnit/parity/tier2/skill_prompt_snapshot.md`: + - If `.claude/skills/darnit-audit/` exists in the repo: copy its system-prompt content verbatim. + - If the skill lives outside the repo (Claude Code user-scope): document the source and commit the snapshot as of the date the parity feature ships. + - Add a top-of-file comment stating "SNAPSHOT of the /darnit-audit skill prompt as of . If the live skill changes, this snapshot MAY drift; re-capture as a routine maintenance task." + +### GitHub Actions workflow + its own tests + +- [X] T023 [US2] Create `.github/workflows/parity-tier2.yml` per contract `tier2-workflow.md` (T2-1..T2-16): + - `on: workflow_dispatch:` with a single `fixture_glob` input (default `"*"`). + - Job runs on `ubuntu-latest` with `environment: parity-tier2` and `permissions: contents: read` only. + - Steps: checkout, setup-python, uv sync --dev, preflight-log (actor + SHA + timestamp to $GITHUB_STEP_SUMMARY), run `uv run python tests/darnit/parity/tier2/run.py --fixture-glob "${{ inputs.fixture_glob }}"` with `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}`, upload `parity-artifacts/` via `actions/upload-artifact@v4` with `if: always()` so artifacts land on any exit code. + - Add YAML comments documenting each T2-* rule this line satisfies for future reviewers. + +- [X] T024 [P] [US2] Create `tests/darnit/parity/tier2/test_workflow_config.py` (offline Tier 1-style test enforcing the workflow's governance shape): + - T2-1: parse `.github/workflows/parity-tier2.yml` as YAML; assert `on` keys are exactly `["workflow_dispatch"]`. + - T2-2: assert job declares `environment: parity-tier2` (exact case-sensitive string). + - T2-5: assert job declares `permissions.contents == "read"` and no other permissions are granted. + - T2-10: assert workflow does NOT accept an `api_key` input (governance regression guard). + - SC-005a + T2-4 (LC2 fix -- portable, no subprocess `grep`): iterate `Path(".github/workflows").glob("*.yml")` and `.glob("*.yaml")`; for each file, assert either the file's `name == "parity-tier2.yml"` OR the substring `"ANTHROPIC_API_KEY"` is absent from `file.read_text()`. Pure Python; works on Linux, macOS, Windows CI equally. + - T2-11: assert an `actions/upload-artifact` step with `if: always()`. + +### Adversarial-response Tier 2 tests (offline) + +- [X] T025 [P] [US2] Create `tests/darnit/parity/tier2/test_diff_adversarial.py`: + - SC-004: feed `diff()` a hand-built `SkillReport(parseable=True, controls=[Control(id="X", status="PASS", ...)])` and an `AuditResult` with the same control at status `"WARN"`; assert diff returns `PER_CONTROL_DISAGREE` outcome, includes X in the failing controls list. + - Suggestive-authority case: same setup but the control's authority is `"suggestive"`; assert diff STILL returns `PER_CONTROL_DISAGREE` (T2 has no license to reinterpret regardless of authority; per FR-008). + - Unparseable case: feed `SkillReport(parseable=False, raw_markdown="...")`; assert diff returns `SKILL_UNPARSEABLE` outcome. + - Counts-only disagreement: feed a `SkillReport` where the summary counts differ from the tool but per-control claims agree; assert `COUNTS_DISAGREE` outcome. + - **FR-010 fail-fast (MC1 fix)**: `test_missing_api_key_raises_setup_error` -- with `monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)`, instantiate `ClaudeAgentSdkClient` and call `invoke_skill(...)`; assert `SetupError` (or the equivalent named exception from T018) is raised with the substring `ANTHROPIC_API_KEY` in the message. Additionally: run `python tests/darnit/parity/tier2/run.py --fixture-glob "*" --dry-run=false` in a subprocess with the env var stripped (`env={"PATH": os.environ["PATH"]}`), assert exit code is 3 (SETUP per contract T2-13). + +**Checkpoint**: Tier 2 machinery is complete offline. A `--dry-run` invocation of `run.py` writes artifacts against a stubbed SDK response. The GitHub Actions workflow YAML is present and passes its own config tests. Merge unblocks a manual `workflow_dispatch` invocation by an authorized maintainer. + +--- + +## Phase 5: User Story 3 -- Fixture auto-discovery (P3) + +**Goal**: A maintainer adds a new fixture directory under `tests/darnit/parity/fixtures/` and the parity tests automatically include it on the next run -- no test file changes required. + +**Independent Test**: Add a directory to `tests/darnit/parity/fixtures/`, run pytest, observe the new test IDs. + +### Test for US3 + +- [X] T026 [P] [US3] Create `tests/darnit/parity/tier1/test_auto_discovery.py`: + - SC-007: Programmatically create a temporary fixture in a subdirectory of `tests/darnit/parity/fixtures/` (using `monkeypatch` or a helper that tracks the addition), rerun collection via `pytest.main(["--collect-only", ...])`, assert the new fixture's test ID appears in the collected tests, then clean up. + - Alternative shape (if temp-fixture creation is fragile): assert the existing fixture count in `test_mcp_vs_harness.py::test_parity` equals the number of fixture directories that contain `.baseline.toml`, with no manual list to keep in sync. + +**Checkpoint**: SC-007 verified. US3 layers cleanly on US1's infrastructure; if T012 (auto-discovery conftest) is done correctly, this test is largely a formality. + +--- + +## Phase 6: Polish & Cross-Cutting + +**Purpose**: Docs, CI wiring, final sanity sweep, PR bookkeeping. + +- [X] T027 [P] Update `CLAUDE.md`'s "Recent Changes" section (top of list) with a one-paragraph 028 entry describing the two-tier parity suite, closes #366, governance-gated Tier 2. + +- [X] T028 [P] Run `uv run ruff check .` and `uv run ruff format --check .` on the new test files. Fix any lint issues. + +- [X] T029 [P] Run `uv run python scripts/validate_sync.py --verbose` -- this feature doesn't touch product code but keep the check honest. + +- [X] T030 [P] Full test sweep: `uv run pytest tests/ -q`. Expected: previous baseline + ~15-25 new tests (T004-T007 + T013-T015 + T017 + T024-T026), all pass, no regressions. + +- [ ] T031 [P] Grep sanity: `grep -r "ANTHROPIC_API_KEY" .github/workflows/` returns matches ONLY in `parity-tier2.yml`. Run manually as pre-PR verification of SC-005a; T024 codifies this as a test but a manual check confirms the CI setup end-to-end. + +- [X] T031a [P] **MC3 fix**: Create `tests/darnit/parity/tier1/test_no_product_changes.py` enforcing FR-014 mechanically: + - Detect base ref via `git rev-parse --verify origin/main 2>/dev/null` (fall back to `main`); on local dev where no base is reachable, skip with `pytest.skip("no base ref -- CI-only check")`. + - Run `git diff --name-only ...HEAD` and collect the file list. + - Assert NO file in the diff is under `packages/darnit/src/` OR `packages/darnit-baseline/src/`. + - Exempts test files (`packages/*/tests/`) and package config (`packages/*/pyproject.toml`) so a legitimate build-config touch isn't blocked; SC-006's product-dep check runs separately. + - If a violation is detected, the assertion message lists the offending files with a note pointing at FR-014. + - This test guards against future maintainers accidentally adding "a small helper" to `packages/darnit/src/darnit/harness/` from a parity-tests PR. + +- [ ] T032 Manual Tier 2 dry-run against the local repo: `ANTHROPIC_API_KEY=... uv run python tests/darnit/parity/tier2/run.py --fixture-glob "all_pass_repo"` on your workstation, then inspect `parity-artifacts/`. Verify the artifact bundle shape matches T2-11 / data-model.md section 7. + +- [ ] T033 Write the PR description. Structure per project convention: no Co-Authored-By: Claude trailer, no Generated with Claude Code footer. Include a summary, the governance rationale for Tier 2 manual-only, test plan, links to spec/plan/contracts, cross-links to #366 (close) + #368 + #369 (related). + +--- + +## Dependencies & Story Completion Order + +``` +Phase 1 (T001-T003) --setup-- + | + v +Phase 2 (T004-T007) --foundational: comparator + fixture_meta-- + | + +------------+----------------------+ + v v v + Phase 3 (T008-T015) Phase 4 (T016-T025) + US1 -- MVP (Tier 1) US2 (Tier 2 machinery + workflow) + | + v + Phase 5 (T026) + US3 -- auto-discovery test + | + v + Phase 6 (T027-T033) --polish-- +``` + +- **Phase 1**: T001 first (scaffold); T002 [P], T003 [P] parallelizable after T001. +- **Phase 2**: T004 first (comparator module -- others depend on `AuditResult` shape); T005 [P] can start alongside T004 since it doesn't import comparator; T006 and T007 are [P] tests after their subjects exist. +- **Phase 3**: T008-T011 (fixtures) are all [P] with each other. T012 depends on T004 + T005. T013-T015 depend on T012 + at least one fixture. +- **Phase 4**: T016, T017, T018, T019, T022 are all [P] with each other. T020 depends on T016 + T018 (uses SkillReport + SDK client). T021 depends on T020 + T019. T023 depends on T021 (references `run.py`). T024, T025 are [P] tests. +- **Phase 5**: T026 depends on Phase 3 being complete. +- **Phase 6**: T030 depends on ALL previous. T027-T029, T031-T032 are largely [P]. T033 last (needs the full picture). + +## Parallel Execution Examples + +Within Phase 3, once Phase 2 is done: + +```bash +# Fixtures in parallel +mkdir -p tests/darnit/parity/fixtures/{all_pass_repo,all_fail_repo,mixed_repo,pending_llm_repo} +# ... populate each ... + +# Then tests in parallel +uv run pytest tests/darnit/parity/tier1/test_mcp_vs_harness.py \ + tests/darnit/parity/tier1/test_comparator_adversarial.py \ + tests/darnit/parity/tier1/test_corpus_inventory.py \ + -q -n auto +``` + +## Implementation Strategy + +**MVP-first order**: Phase 1 -> Phase 2 -> Phase 3 (Tier 1 ships as its own PR increment if we want smaller reviews). Tier 2 (Phase 4) layers on top without changing Tier 1. Phase 5 is a small verification test; Phase 6 is polish. + +**Two-PR option**: If the review surface for one PR is too big, split as: +- PR A: Phases 1 + 2 + 3 + 5 + partial polish = "Tier 1 MCP-vs-harness parity" +- PR B: Phase 4 + remaining polish = "Tier 2 skill-vs-tool parity" + +Both PRs close #366 partially; the second one carries the actual `Fixes #366` marker. + +**Time boxing**: Phase 3 is the largest slice (~8 tasks, ~4 fixtures + ~4 tests). Phase 4 is comparable (~10 tasks, more machinery). Total estimated size: ~600-800 lines net production + ~600-800 lines tests. Comparable to feature 027. + +## Test coverage matrix + +| Success Criterion / FR | Test task(s) | +|---|---| +| SC-001 (Tier 1 catches adversarial divergence) | T014 (comparator adversarial) | +| SC-002 (Tier 1 <60s) | T013 (`pytest.mark.timeout(60)`) | +| SC-003 (every drift has a row) | T014 (5-divergence assertion) | +| SC-004 (Tier 2 catches skill reclassification, regardless of authority) | T025 (diff adversarial + suggestive-authority case) | +| SC-005 (Tier 2 artifacts have both skill + tool on failure) | T019 (artifact_writer) + T021 (integration) | +| SC-005a (no ANTHROPIC_API_KEY exposure) | T024 (workflow config test, portable Python file iteration per LC2) + T031 (manual grep) | +| SC-006 (no product deps added) | T002 (workspace dev group only) + manual pyproject.toml diff | +| SC-007 (fixture auto-discovery) | T012 (conftest) + T026 (auto-discovery test) | +| SC-008 (four fixture categories) | T015 (corpus inventory) | +| SC-009 (issue #366 closed) | T033 (PR description with `Fixes #366`) | +| FR-010 (missing key fail-fast) | T025 (dedicated subtest -- monkeypatch.delenv + subprocess exit code 3 assertion) | +| FR-013 (green-run evidence emitted) | T013 (capsys assertion on the summary-line pattern) | +| FR-014 (no product code changes) | T031a (git-diff-based mechanical enforcement) | +| FR-015 (Tier 1 deterministic) | T006 (run-twice byte-identical assertion) | diff --git a/tests/darnit/parity/__init__.py b/tests/darnit/parity/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/darnit/parity/fixtures/all_fail_repo/.baseline.toml b/tests/darnit/parity/fixtures/all_fail_repo/.baseline.toml new file mode 100644 index 0000000..d5e4c9d --- /dev/null +++ b/tests/darnit/parity/fixtures/all_fail_repo/.baseline.toml @@ -0,0 +1,8 @@ +extends = "openssf-baseline" + +# Feature 028 parity fixture: expects every selected control to FAIL. +# Same controls as all_pass_repo but the fixture omits the required files +# so file_exists returns FAIL. +[audit_profiles.parity_subset] +description = "Feature 028 parity fixture: 2 dispositive controls, all missing" +controls = ["OSPS-DO-01.01", "OSPS-LE-03.01"] diff --git a/tests/darnit/parity/fixtures/all_fail_repo/.gitkeep b/tests/darnit/parity/fixtures/all_fail_repo/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/darnit/parity/fixtures/all_fail_repo/parity.toml b/tests/darnit/parity/fixtures/all_fail_repo/parity.toml new file mode 100644 index 0000000..97cdd62 --- /dev/null +++ b/tests/darnit/parity/fixtures/all_fail_repo/parity.toml @@ -0,0 +1,13 @@ +[expected] +category = "all_fail" +has_pending_llm = false +strict = false +control_ids = ["OSPS-DO-01.01", "OSPS-LE-03.01"] + +[expected.counts] +pass = 0 +fail = 2 +warn = 0 +error = 0 +n_a = 0 +pending_llm = 0 diff --git a/tests/darnit/parity/fixtures/all_pass_repo/.baseline.toml b/tests/darnit/parity/fixtures/all_pass_repo/.baseline.toml new file mode 100644 index 0000000..52dcc33 --- /dev/null +++ b/tests/darnit/parity/fixtures/all_pass_repo/.baseline.toml @@ -0,0 +1,7 @@ +extends = "openssf-baseline" + +# Feature 028 parity fixture: expects every selected control to PASS. +# Uses two simple dispositive file_exists controls so no LLM involvement. +[audit_profiles.parity_subset] +description = "Feature 028 parity fixture: 2 dispositive controls" +controls = ["OSPS-DO-01.01", "OSPS-LE-03.01"] diff --git a/tests/darnit/parity/fixtures/all_pass_repo/LICENSE b/tests/darnit/parity/fixtures/all_pass_repo/LICENSE new file mode 100644 index 0000000..354eb9c --- /dev/null +++ b/tests/darnit/parity/fixtures/all_pass_repo/LICENSE @@ -0,0 +1,17 @@ +MIT License + +Copyright (c) 2026 darnit parity fixture + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. diff --git a/tests/darnit/parity/fixtures/all_pass_repo/README.md b/tests/darnit/parity/fixtures/all_pass_repo/README.md new file mode 100644 index 0000000..bb4d564 --- /dev/null +++ b/tests/darnit/parity/fixtures/all_pass_repo/README.md @@ -0,0 +1,14 @@ +# all-pass parity fixture + +A minimal repository used by feature 028's Tier 1 parity tests. Every +selected control is expected to PASS under both the MCP tool path and the +harness path. + +## Contents + +- `README.md` -- satisfies `OSPS-DO-01.01` (HasReadme) +- `LICENSE` -- satisfies `OSPS-LE-03.01` (LicenseInRepo) + +## Not for humans + +Do not modify this fixture without also updating `parity.toml`. diff --git a/tests/darnit/parity/fixtures/all_pass_repo/parity.toml b/tests/darnit/parity/fixtures/all_pass_repo/parity.toml new file mode 100644 index 0000000..ed2a72d --- /dev/null +++ b/tests/darnit/parity/fixtures/all_pass_repo/parity.toml @@ -0,0 +1,13 @@ +[expected] +category = "all_pass" +has_pending_llm = false +strict = false +control_ids = ["OSPS-DO-01.01", "OSPS-LE-03.01"] + +[expected.counts] +pass = 2 +fail = 0 +warn = 0 +error = 0 +n_a = 0 +pending_llm = 0 diff --git a/tests/darnit/parity/fixtures/mixed_repo/.baseline.toml b/tests/darnit/parity/fixtures/mixed_repo/.baseline.toml new file mode 100644 index 0000000..087a0e0 --- /dev/null +++ b/tests/darnit/parity/fixtures/mixed_repo/.baseline.toml @@ -0,0 +1,7 @@ +extends = "openssf-baseline" + +# Feature 028 parity fixture: some controls PASS, some FAIL. Neither +# control involves an LLM step. +[audit_profiles.parity_subset] +description = "Feature 028 parity fixture: mixed pass/fail" +controls = ["OSPS-DO-01.01", "OSPS-LE-03.01"] diff --git a/tests/darnit/parity/fixtures/mixed_repo/README.md b/tests/darnit/parity/fixtures/mixed_repo/README.md new file mode 100644 index 0000000..22317ef --- /dev/null +++ b/tests/darnit/parity/fixtures/mixed_repo/README.md @@ -0,0 +1,4 @@ +# mixed parity fixture + +Has a README (satisfies `OSPS-DO-01.01`) but intentionally omits the +LICENSE file (fails `OSPS-LE-03.01`). Produces one PASS and one FAIL. diff --git a/tests/darnit/parity/fixtures/mixed_repo/parity.toml b/tests/darnit/parity/fixtures/mixed_repo/parity.toml new file mode 100644 index 0000000..8be3844 --- /dev/null +++ b/tests/darnit/parity/fixtures/mixed_repo/parity.toml @@ -0,0 +1,13 @@ +[expected] +category = "mixed" +has_pending_llm = false +strict = false +control_ids = ["OSPS-DO-01.01", "OSPS-LE-03.01"] + +[expected.counts] +pass = 1 +fail = 1 +warn = 0 +error = 0 +n_a = 0 +pending_llm = 0 diff --git a/tests/darnit/parity/fixtures/pending_llm_repo/.baseline.toml b/tests/darnit/parity/fixtures/pending_llm_repo/.baseline.toml new file mode 100644 index 0000000..3ccd838 --- /dev/null +++ b/tests/darnit/parity/fixtures/pending_llm_repo/.baseline.toml @@ -0,0 +1,10 @@ +extends = "openssf-baseline" + +# Feature 028 parity fixture: exercises the LLM path via STAGE1-REF-SECURITY-01. +# Under the MCP tool path (stop_on_llm=True), this control's llm_extract +# step stops the pipeline and the result is PENDING_LLM. Under the harness +# path with MockLLMStep=inconclusive, it resolves to WARN via the +# verify_with_llm_response fallthrough. That is the sole allowed drift. +[audit_profiles.parity_subset] +description = "Feature 028 parity fixture: LLM-required control" +tags = { "stage1-ref" = true } diff --git a/tests/darnit/parity/fixtures/pending_llm_repo/README.md b/tests/darnit/parity/fixtures/pending_llm_repo/README.md new file mode 100644 index 0000000..6d13e00 --- /dev/null +++ b/tests/darnit/parity/fixtures/pending_llm_repo/README.md @@ -0,0 +1,10 @@ +# pending-LLM parity fixture + +Exercises `STAGE1-REF-SECURITY-01`, whose first pass is a suggestive +`llm_extract` step. Under the MCP tool path (which never dispatches), +this control is left PENDING_LLM. Under the harness path with a +`MockLLMStep` returning `inconclusive`, it resolves to WARN. That +divergence is the sole documented allowed drift for Tier 1. + +The README exists so the llm_extract step has content to reference. +No `SECURITY.md`, so a dispositive file_exists would FAIL if reached. diff --git a/tests/darnit/parity/fixtures/pending_llm_repo/parity.toml b/tests/darnit/parity/fixtures/pending_llm_repo/parity.toml new file mode 100644 index 0000000..f398f47 --- /dev/null +++ b/tests/darnit/parity/fixtures/pending_llm_repo/parity.toml @@ -0,0 +1,13 @@ +[expected] +category = "pending_llm" +has_pending_llm = true +strict = false +control_ids = ["STAGE1-REF-SECURITY-01"] + +[expected.counts] +pass = 0 +fail = 0 +warn = 0 +error = 0 +n_a = 0 +pending_llm = 1 diff --git a/tests/darnit/parity/tier1/__init__.py b/tests/darnit/parity/tier1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/darnit/parity/tier1/comparator.py b/tests/darnit/parity/tier1/comparator.py new file mode 100644 index 0000000..7e5fc68 --- /dev/null +++ b/tests/darnit/parity/tier1/comparator.py @@ -0,0 +1,268 @@ +"""Comparator for Tier 1 audit parity (feature 028 T004). + +Diffs two `AuditResult` instances (one from the direct MCP tool call, one +from the harness) and produces a `ParityReport` with a drift table. + +The single allowed drift class is: MCP tool leaves a control PENDING_LLM, +harness resolves it via its LLM continuation loop to any non-PENDING_LLM +status. Any other divergence is a hard failure. + +See: + - specs/028-audit-parity-tests/data-model.md sections 3-5 + - specs/028-audit-parity-tests/contracts/tier1-parity-invariant.md +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from darnit.harness.report import HarnessReport + +Status = Literal["PASS", "FAIL", "WARN", "N/A", "ERROR", "PENDING_LLM"] +STATUSES: tuple[Status, ...] = ( + "PASS", + "FAIL", + "WARN", + "N/A", + "ERROR", + "PENDING_LLM", +) + + +@dataclass(frozen=True) +class Control: + id: str + status: Status + authority: Literal["dispositive", "suggestive", "asserted"] | None = None + level: int | None = None + + +@dataclass(frozen=True) +class AuditResult: + """Normalized shape both paths reduce to for comparison.""" + + controls: tuple[Control, ...] + source: Literal["mcp_tool", "harness"] + + @classmethod + def from_mcp_json(cls, payload: dict[str, Any]) -> AuditResult: + """Parse audit_openssf_baseline(output_format='json') output.""" + results = payload.get("results", []) + return cls( + controls=tuple( + Control( + id=str(r.get("id", "")), + status=r.get("status", "ERROR"), + authority=r.get("authority"), + level=r.get("level"), + ) + for r in results + ), + source="mcp_tool", + ) + + @classmethod + def from_harness_report(cls, report: HarnessReport) -> AuditResult: + """Reduce a HarnessReport (feature 026) to the same shape.""" + return cls( + controls=tuple( + Control( + id=str(c.get("id", "")), + status=c.get("status", "ERROR"), + authority=c.get("authority"), + level=c.get("level"), + ) + for c in report.controls + ), + source="harness", + ) + + def filter_to(self, control_ids: tuple[str, ...] | list[str]) -> AuditResult: + """Return a copy containing only the controls whose id is in the set. + + Neither audit path today applies `audit_profiles` from a fixture's + `.baseline.toml` automatically, so both paths run every OpenSSF + Baseline control. This helper narrows to the fixture-declared + `control_ids` so a fixture can assert parity over its subset without + the noise of unrelated controls. + + If `control_ids` is empty, returns self (no filtering). + """ + if not control_ids: + return self + allowed = set(control_ids) + return AuditResult( + controls=tuple(c for c in self.controls if c.id in allowed), + source=self.source, + ) + + +@dataclass(frozen=True) +class DriftEntry: + """One divergence between the MCP tool and harness paths. + + Statuses that AGREE do not produce a DriftEntry; only divergences do. + A separate flag distinguishes disallowed (hard failure) from allowed + (evidence-only) drift. + """ + + fixture_name: str + control_id: str + mcp_status: str + harness_status: str + note: str = "" # e.g. "missing on harness side", "missing on mcp side" + + @property + def is_allowed_drift(self) -> bool: + """T1-8 canonical table: only PENDING_LLM (MCP) -> a resolved + status on the harness side is allowed. A missing control on + either side is always a HARD failure per T1-3, even if the + other side reads PENDING_LLM. PR #370 review fix. + """ + if self.mcp_status == "" or self.harness_status == "": + return False + if self.mcp_status == "PENDING_LLM" and self.harness_status != "PENDING_LLM": + return True + return False + + +@dataclass(frozen=True) +class ParityReport: + fixture_name: str + total_controls: int + agreements: int + drifts: tuple[DriftEntry, ...] + + @property + def disallowed_drifts(self) -> tuple[DriftEntry, ...]: + return tuple(d for d in self.drifts if not d.is_allowed_drift) + + @property + def allowed_drifts(self) -> tuple[DriftEntry, ...]: + return tuple(d for d in self.drifts if d.is_allowed_drift) + + @property + def is_green(self) -> bool: + return len(self.disallowed_drifts) == 0 + + def format_summary_line(self) -> str: + """FR-013 evidence line, emitted on every run.""" + return ( + f"[tier1] {self.fixture_name}: " + f"{self.total_controls} controls compared, " + f"{self.agreements} agreed, " + f"{len(self.disallowed_drifts)} diverged, " + f"{len(self.allowed_drifts)} allowed-drift" + ) + + def format_failure_table(self) -> str: + """FR-004: fixed-width Markdown table (no ANSI) for pytest messages. + + Called when disallowed_drifts is non-empty. If called on a green + report, produces "No disallowed drifts." + """ + disallowed = self.disallowed_drifts + if not disallowed: + return "No disallowed drifts." + + headers = ["control_id", "mcp_status", "harness_status", "note"] + rows = [[d.control_id, d.mcp_status, d.harness_status, d.note] for d in disallowed] + + # Compute column widths (max of header and any row value). + widths = [max(len(headers[i]), *(len(row[i]) for row in rows)) for i in range(len(headers))] + + def _fmt_row(row: list[str]) -> str: + cells = [row[i].ljust(widths[i]) for i in range(len(row))] + return "| " + " | ".join(cells) + " |" + + separator = "| " + " | ".join("-" * w for w in widths) + " |" + lines = [ + f"Fixture: {self.fixture_name}", + f"Disallowed drifts: {len(disallowed)}", + "", + _fmt_row(headers), + separator, + *[_fmt_row(row) for row in rows], + ] + return "\n".join(lines) + + +def compare( + mcp: AuditResult, + harness: AuditResult, + fixture_name: str, +) -> ParityReport: + """Diff two AuditResults per the T1-8 allowed-drift table. + + - Statuses that agree: no DriftEntry produced. + - Controls present on one side but not the other: DriftEntry with + note='missing on '; treated as disallowed (T1-3). + - Status divergences: DriftEntry; classification via is_allowed_drift. + """ + mcp_by_id = {c.id: c for c in mcp.controls} + harness_by_id = {c.id: c for c in harness.controls} + + all_ids = sorted(set(mcp_by_id) | set(harness_by_id)) + drifts: list[DriftEntry] = [] + agreements = 0 + + for cid in all_ids: + m = mcp_by_id.get(cid) + h = harness_by_id.get(cid) + + if m is None and h is not None: + drifts.append( + DriftEntry( + fixture_name=fixture_name, + control_id=cid, + mcp_status="", + harness_status=h.status, + note="missing on mcp side", + ) + ) + continue + if h is None and m is not None: + drifts.append( + DriftEntry( + fixture_name=fixture_name, + control_id=cid, + mcp_status=m.status, + harness_status="", + note="missing on harness side", + ) + ) + continue + + # Both present. + assert m is not None and h is not None + if m.status == h.status: + agreements += 1 + else: + drifts.append( + DriftEntry( + fixture_name=fixture_name, + control_id=cid, + mcp_status=m.status, + harness_status=h.status, + ) + ) + + return ParityReport( + fixture_name=fixture_name, + total_controls=len(all_ids), + agreements=agreements, + drifts=tuple(drifts), + ) + + +__all__ = ( + "Control", + "AuditResult", + "DriftEntry", + "ParityReport", + "Status", + "STATUSES", + "compare", +) diff --git a/tests/darnit/parity/tier1/conftest.py b/tests/darnit/parity/tier1/conftest.py new file mode 100644 index 0000000..612bdd4 --- /dev/null +++ b/tests/darnit/parity/tier1/conftest.py @@ -0,0 +1,154 @@ +"""Tier 1 conftest: fixture auto-discovery + prepared-fixture helper (T012). + +Auto-discovers `tests/darnit/parity/fixtures//.baseline.toml` and +parametrizes `fixture_dir`. Provides `prepared_fixture`, `mcp_tool_result`, +and `harness_result` pytest fixtures that materialize a git-initialized +copy of the fixture and run both audit paths against it. + +HC1 fix: fixtures are copied to `tmp_path` and git-initialized before +either path is invoked, because `HarnessRun._initial_audit` -> +`prepare_audit` -> `detect_repo_from_git` requires a git repo. +""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from darnit.core.llm_step import LLMJudgment, MockLLMStep +from darnit.harness.driver import HarnessRun +from darnit_baseline.tools import audit_openssf_baseline +from tests.darnit.parity.tier1.comparator import AuditResult + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +def pytest_collection_modifyitems(config: pytest.Config, items: list) -> None: + """Auto-mark every test in tier1/ as `integration` (PR #370 review fix). + + Parity tests spin up the full harness against a git-initialized + fixture repo, so they never satisfy the `unit` marker's contract. + Without an explicit mark, CI's `-m unit / -m integration` split + silently deselected the whole suite. This hook applies the mark + to every collected item under this conftest. + """ + integration_mark = pytest.mark.integration + for item in items: + item.add_marker(integration_mark) + + +@pytest.fixture(autouse=True) +def _ensure_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + """The harness's credential check requires ANTHROPIC_API_KEY; set a + dummy value so tests that don't exercise missing-key paths run cleanly.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key-not-real") + + +def _discover_fixtures() -> list[Path]: + if not FIXTURES_DIR.exists(): + return [] + return sorted(p for p in FIXTURES_DIR.iterdir() if p.is_dir() and (p / ".baseline.toml").exists()) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Auto-discover fixtures for any test that requests `fixture_dir`.""" + if "fixture_dir" in metafunc.fixturenames: + fixtures = _discover_fixtures() + metafunc.parametrize( + "fixture_dir", + fixtures, + ids=[f.name for f in fixtures], + ) + + +@pytest.fixture +def prepared_fixture( + fixture_dir: Path, + tmp_path: Path, +) -> Path: + """Copy the fixture into tmp_path and git-init it (HC1). + + HarnessRun and audit_openssf_baseline both call prepare_audit which + requires the target directory to be a git repo. Static fixture dirs + under tests/darnit/parity/fixtures/ are NOT git-initialized, so we + copy them to a tmp_path and initialize there. + """ + dest = tmp_path / fixture_dir.name + shutil.copytree(fixture_dir, dest) + subprocess.run( + ["git", "init", "--initial-branch=main", "-q"], + cwd=dest, + check=True, + capture_output=True, + ) + subprocess.run( + [ + "git", + "-c", + "user.name=test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "-q", + "-m", + "init", + ], + cwd=dest, + check=True, + capture_output=True, + ) + subprocess.run( + [ + "git", + "remote", + "add", + "origin", + "https://github.com/fake-owner/fake-repo.git", + ], + cwd=dest, + check=True, + capture_output=True, + ) + return dest + + +@pytest.fixture +def mcp_tool_result(prepared_fixture: Path) -> AuditResult: + """Invoke audit_openssf_baseline directly; return normalized AuditResult.""" + raw = audit_openssf_baseline( + local_path=str(prepared_fixture), + level=3, + output_format="json", + auto_init_config=False, + attest=False, + prefer_upstream=False, + ) + payload = json.loads(raw) + return AuditResult.from_mcp_json(payload) + + +@pytest.fixture +def harness_result(prepared_fixture: Path) -> AuditResult: + """Run HarnessRun with MockLLMStep=inconclusive; return normalized AuditResult.""" + mock = MockLLMStep( + LLMJudgment( + outcome="inconclusive", + confidence=0.0, + reasoning="tier1-mock: no LLM decision", + ) + ) + run = HarnessRun( + local_path=str(prepared_fixture), + level=3, + llm_step=mock, + per_call_timeout_s=5, + total_run_timeout_s=30, + ) + report = asyncio.new_event_loop().run_until_complete(run.run()) + return AuditResult.from_harness_report(report) diff --git a/tests/darnit/parity/tier1/fixture_meta.py b/tests/darnit/parity/tier1/fixture_meta.py new file mode 100644 index 0000000..c1573b4 --- /dev/null +++ b/tests/darnit/parity/tier1/fixture_meta.py @@ -0,0 +1,179 @@ +"""parity.toml metadata parser for feature 028 fixtures (T005). + +Per contract parity-toml-schema.md (PT-1..PT-19). TOML-parsed via stdlib +tomllib; no code execution at load time. +""" + +from __future__ import annotations + +import tomllib +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + +Category = Literal["all_pass", "all_fail", "mixed", "pending_llm"] +VALID_CATEGORIES: tuple[Category, ...] = ( + "all_pass", + "all_fail", + "mixed", + "pending_llm", +) + +_KNOWN_COUNT_KEYS = {"pass", "fail", "warn", "error", "n_a", "pending_llm"} +_KNOWN_EXPECTED_KEYS = { + "category", + "has_pending_llm", + "strict", + "counts", + "controls", + "control_ids", +} + + +@dataclass(frozen=True) +class ExpectedControl: + id: str + status: str + + +@dataclass(frozen=True) +class ParityMetadata: + category: Category + has_pending_llm: bool + strict: bool = False + counts: dict[str, int] = field(default_factory=dict) + controls: tuple[ExpectedControl, ...] = () + # Optional per-fixture filter: if non-empty, the parity comparison + # only considers these control IDs from both paths' outputs. If empty, + # all controls (which will be every OSPS control since neither path + # auto-applies audit_profiles from .baseline.toml today) are compared. + control_ids: tuple[str, ...] = () + + +def load_parity_metadata(fixture_dir: Path) -> ParityMetadata | None: + """Load a fixture's parity.toml, if present. + + Returns None when the file is absent (PT-2). Raises ValueError on + malformed TOML (PT-4) or schema violations (PT-5, PT-6, PT-8). + Unknown keys log a warning but do not fail (PT-9). + """ + parity_path = fixture_dir / "parity.toml" + if not parity_path.exists(): + return None + + try: + with parity_path.open("rb") as f: + raw = tomllib.load(f) + except tomllib.TOMLDecodeError as exc: + raise ValueError( + f"malformed parity.toml at {parity_path}: {exc}", + ) from exc + + expected = raw.get("expected") + if not isinstance(expected, dict): + raise ValueError( + f"parity.toml at {parity_path} is missing [expected] section", + ) + + # Forward-compat: unknown [expected] keys warn but don't fail (PT-9). + for key in expected: + if key not in _KNOWN_EXPECTED_KEYS: + warnings.warn( + f"parity.toml at {parity_path}: unknown [expected] key {key!r}", + stacklevel=2, + ) + + # Validate category (PT-5). + category = expected.get("category") + if category not in VALID_CATEGORIES: + raise ValueError( + f"parity.toml at {parity_path}: `category` must be one of {VALID_CATEGORIES}, got {category!r}", + ) + + # Validate counts (PT-8). + counts_raw = expected.get("counts", {}) or {} + if not isinstance(counts_raw, dict): + raise ValueError( + f"parity.toml at {parity_path}: [expected.counts] must be a table", + ) + counts: dict[str, int] = {} + for k, v in counts_raw.items(): + if k not in _KNOWN_COUNT_KEYS: + warnings.warn( + f"parity.toml at {parity_path}: unknown counts key {k!r}", + stacklevel=2, + ) + continue + if not isinstance(v, int) or v < 0: + raise ValueError( + f"parity.toml at {parity_path}: counts.{k} must be a non-negative integer, got {v!r}", + ) + counts[k] = v + + # Derive / validate has_pending_llm (PT-6). + pending_count = counts.get("pending_llm", 0) + if "has_pending_llm" in expected: + has_pending_llm = expected["has_pending_llm"] + if not isinstance(has_pending_llm, bool): + raise ValueError( + f"parity.toml at {parity_path}: has_pending_llm must be bool", + ) + if bool(pending_count > 0) != has_pending_llm: + raise ValueError( + f"parity.toml at {parity_path}: has_pending_llm=" + f"{has_pending_llm} disagrees with counts.pending_llm=" + f"{pending_count}", + ) + else: + has_pending_llm = pending_count > 0 + + strict = bool(expected.get("strict", False)) + + # Optional per-control expectations. + controls_raw = expected.get("controls", []) or [] + if not isinstance(controls_raw, list): + raise ValueError( + f"parity.toml at {parity_path}: [[expected.controls]] must be an array", + ) + controls: list[ExpectedControl] = [] + for entry in controls_raw: + if not isinstance(entry, dict): + raise ValueError( + f"parity.toml at {parity_path}: control entry must be a table", + ) + cid = entry.get("id") + status = entry.get("status") + if not isinstance(cid, str) or not cid: + raise ValueError( + f"parity.toml at {parity_path}: control entry needs non-empty `id`", + ) + if not isinstance(status, str) or not status: + raise ValueError( + f"parity.toml at {parity_path}: control entry needs non-empty `status`", + ) + controls.append(ExpectedControl(id=cid, status=status)) + + control_ids_raw = expected.get("control_ids", []) or [] + if not isinstance(control_ids_raw, list) or not all(isinstance(x, str) for x in control_ids_raw): + raise ValueError( + f"parity.toml at {parity_path}: control_ids must be a list of strings", + ) + + return ParityMetadata( + category=category, + has_pending_llm=has_pending_llm, + strict=strict, + counts=counts, + controls=tuple(controls), + control_ids=tuple(control_ids_raw), + ) + + +__all__ = ( + "Category", + "VALID_CATEGORIES", + "ExpectedControl", + "ParityMetadata", + "load_parity_metadata", +) diff --git a/tests/darnit/parity/tier1/test_auto_discovery.py b/tests/darnit/parity/tier1/test_auto_discovery.py new file mode 100644 index 0000000..0339dff --- /dev/null +++ b/tests/darnit/parity/tier1/test_auto_discovery.py @@ -0,0 +1,60 @@ +"""SC-007 fixture auto-discovery test (feature 028 T026). + +Asserts that fixture count matches directory count -- when a maintainer +adds a directory under `tests/darnit/parity/fixtures/`, the parity test +suite includes it on the next collection without any test file edit. +""" + +from __future__ import annotations + +from pathlib import Path + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +def _discovered_fixture_dirs() -> list[Path]: + if not FIXTURES_DIR.exists(): + return [] + return sorted(p for p in FIXTURES_DIR.iterdir() if p.is_dir() and (p / ".baseline.toml").exists()) + + +def test_fixture_count_matches_directory_count() -> None: + """SC-007: no manual list of fixtures in test code -- discovery is + directory-driven. Adding or removing a fixture only requires touching + the fixture's directory.""" + fixtures = _discovered_fixture_dirs() + # Sanity: at least the four MVP fixtures. + assert len(fixtures) >= 4, f"Expected at least 4 fixtures, got {len(fixtures)}: {[f.name for f in fixtures]}" + # Every discovered directory contains a `.baseline.toml` -- the + # required marker file. If a maintainer adds a directory without + # `.baseline.toml`, it's silently ignored (not counted as a fixture). + for fixture in fixtures: + assert (fixture / ".baseline.toml").exists() + + +def test_new_fixture_is_picked_up() -> None: + """Directly exercise the discovery function with a synthetic addition. + + Uses a temporary side directory to avoid mutating the real corpus; + verifies the discovery pattern would include it. + """ + import shutil + import tempfile + + # Simulate the discovery by pointing at a temp copy of the fixtures dir + # plus one extra fake fixture. + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + for existing in _discovered_fixture_dirs(): + shutil.copytree(existing, tmp_root / existing.name) + + # Add a synthetic fixture. + new_dir = tmp_root / "synthetic_extra" + new_dir.mkdir() + (new_dir / ".baseline.toml").write_text('extends = "openssf-baseline"\n') + + # Rediscover using the same pattern. + discovered = sorted(p for p in tmp_root.iterdir() if p.is_dir() and (p / ".baseline.toml").exists()) + names = {p.name for p in discovered} + assert "synthetic_extra" in names + assert len(discovered) == len(_discovered_fixture_dirs()) + 1 diff --git a/tests/darnit/parity/tier1/test_comparator.py b/tests/darnit/parity/tier1/test_comparator.py new file mode 100644 index 0000000..9aef25d --- /dev/null +++ b/tests/darnit/parity/tier1/test_comparator.py @@ -0,0 +1,201 @@ +"""Tests for the Tier 1 comparator (feature 028 T006). + +Covers: + - T1-9 mechanical enumeration: all 36 (mcp_status, harness_status) pairs + classified per the T1-8 canonical drift table. + - AuditResult factories (from_mcp_json + from_harness_report) shape. + - FR-004: format_failure_table produces fixed-width Markdown with no ANSI. + - FR-013: format_summary_line shape. + - FR-015 / MC4: determinism -- run compare() twice, assert byte-identical + output. +""" + +from __future__ import annotations + +from tests.darnit.parity.tier1.comparator import ( + STATUSES, + AuditResult, + Control, + DriftEntry, + ParityReport, + compare, +) + + +def _make_mcp(controls: list[Control]) -> AuditResult: + return AuditResult(controls=tuple(controls), source="mcp_tool") + + +def _make_harness(controls: list[Control]) -> AuditResult: + return AuditResult(controls=tuple(controls), source="harness") + + +class TestAllowedDriftTable: + """T1-9: enumerate every (mcp, harness) status pair and verify + the comparator's classification matches the T1-8 table.""" + + def test_all_36_pairs_classified_correctly(self) -> None: + for m_status in STATUSES: + for h_status in STATUSES: + mcp = _make_mcp([Control(id="X", status=m_status)]) + harness = _make_harness([Control(id="X", status=h_status)]) + report = compare(mcp, harness, "test_fixture") + + if m_status == h_status: + # Agreement -- no drift entry. + assert report.total_controls == 1 + assert report.agreements == 1 + assert len(report.drifts) == 0 + assert report.is_green, f"({m_status}, {h_status}) should be green" + continue + + # Divergence -- exactly one drift. + assert report.total_controls == 1 + assert report.agreements == 0 + assert len(report.drifts) == 1 + drift = report.drifts[0] + assert drift.control_id == "X" + assert drift.mcp_status == m_status + assert drift.harness_status == h_status + + # Classification: PENDING_LLM (MCP) -> non-PENDING_LLM + # (harness) is the sole allowed drift. + if m_status == "PENDING_LLM" and h_status != "PENDING_LLM": + assert drift.is_allowed_drift is True + assert report.is_green, f"PENDING_LLM->{h_status} must be green" + else: + assert drift.is_allowed_drift is False + assert not report.is_green, f"({m_status}, {h_status}) must NOT be green" + + +class TestAuditResultFactories: + def test_from_mcp_json_extracts_fields(self) -> None: + payload = { + "results": [ + { + "id": "OSPS-GV-01.01", + "status": "PASS", + "authority": "dispositive", + "level": 1, + }, + { + "id": "OSPS-BR-06.01", + "status": "FAIL", + "authority": "dispositive", + "level": 2, + }, + ], + } + result = AuditResult.from_mcp_json(payload) + assert result.source == "mcp_tool" + assert len(result.controls) == 2 + assert result.controls[0].id == "OSPS-GV-01.01" + assert result.controls[0].status == "PASS" + assert result.controls[0].authority == "dispositive" + assert result.controls[0].level == 1 + + def test_from_harness_report_equivalent_shape(self) -> None: + """A HarnessReport-shaped input and an MCP-shaped input should + produce equivalent Control values (modulo source).""" + + class _FakeReport: + controls = [ + {"id": "X", "status": "PASS", "authority": "dispositive", "level": 1}, + ] + + harness = AuditResult.from_harness_report(_FakeReport()) + mcp = AuditResult.from_mcp_json( + {"results": [{"id": "X", "status": "PASS", "authority": "dispositive", "level": 1}]}, + ) + assert harness.controls == mcp.controls + assert harness.source == "harness" + assert mcp.source == "mcp_tool" + + +class TestFormatting: + def test_format_failure_table_no_ansi_no_escapes(self) -> None: + """FR-004: fixed-width Markdown; no ANSI escape sequences.""" + report = ParityReport( + fixture_name="test", + total_controls=1, + agreements=0, + drifts=( + DriftEntry( + fixture_name="test", + control_id="X", + mcp_status="PASS", + harness_status="FAIL", + ), + ), + ) + table = report.format_failure_table() + assert "\033" not in table, f"ANSI escape present: {table!r}" + assert "|" in table + assert "X" in table + assert "PASS" in table + assert "FAIL" in table + + def test_format_summary_line_shape(self) -> None: + """FR-013: recognizable evidence-line shape.""" + report = ParityReport( + fixture_name="mixed_repo", + total_controls=10, + agreements=8, + drifts=( + DriftEntry( + fixture_name="mixed_repo", + control_id="A", + mcp_status="PENDING_LLM", + harness_status="WARN", + ), + DriftEntry( + fixture_name="mixed_repo", + control_id="B", + mcp_status="PASS", + harness_status="FAIL", + ), + ), + ) + line = report.format_summary_line() + assert line.startswith("[tier1] mixed_repo:") + assert "10 controls compared" in line + assert "8 agreed" in line + assert "1 diverged" in line + assert "1 allowed-drift" in line + + def test_format_failure_table_on_green_report(self) -> None: + report = ParityReport( + fixture_name="test", + total_controls=1, + agreements=1, + drifts=(), + ) + assert "No disallowed drifts." == report.format_failure_table() + + +class TestDeterminism: + """MC4 / FR-15: repeated runs on identical inputs produce + byte-identical outputs. Guards against dict-iteration-order or + time-dependent regressions.""" + + def test_compare_output_is_deterministic(self) -> None: + mcp = _make_mcp( + [ + Control(id="Z", status="PASS"), + Control(id="A", status="FAIL"), + Control(id="M", status="WARN"), + ] + ) + harness = _make_harness( + [ + Control(id="M", status="PASS"), + Control(id="A", status="FAIL"), + Control(id="Z", status="PASS"), + ] + ) + report1 = compare(mcp, harness, "det_test") + report2 = compare(mcp, harness, "det_test") + + assert report1.format_summary_line() == report2.format_summary_line() + assert report1.format_failure_table() == report2.format_failure_table() + assert report1.drifts == report2.drifts diff --git a/tests/darnit/parity/tier1/test_comparator_adversarial.py b/tests/darnit/parity/tier1/test_comparator_adversarial.py new file mode 100644 index 0000000..73aa6fd --- /dev/null +++ b/tests/darnit/parity/tier1/test_comparator_adversarial.py @@ -0,0 +1,117 @@ +"""Adversarial comparator tests (feature 028 T014). + +Covers SC-001 (comparator catches a seeded PASS-vs-FAIL divergence), +SC-003 (N seeded divergences produce N table rows), and LC1 (all three +PENDING_LLM allowed-drift resolutions). +""" + +from __future__ import annotations + +from tests.darnit.parity.tier1.comparator import ( + AuditResult, + Control, + compare, +) + + +def _mcp(*controls: Control) -> AuditResult: + return AuditResult(controls=tuple(controls), source="mcp_tool") + + +def _harness(*controls: Control) -> AuditResult: + return AuditResult(controls=tuple(controls), source="harness") + + +class TestSC001CatchesDivergence: + def test_pass_to_fail_divergence_flagged(self) -> None: + """SC-001: hand-built PASS vs FAIL produces a disallowed drift.""" + mcp = _mcp(Control(id="X", status="PASS")) + harness = _harness(Control(id="X", status="FAIL")) + report = compare(mcp, harness, "sc001_test") + + assert not report.is_green + assert len(report.disallowed_drifts) == 1 + assert report.disallowed_drifts[0].control_id == "X" + assert report.disallowed_drifts[0].mcp_status == "PASS" + assert report.disallowed_drifts[0].harness_status == "FAIL" + + +class TestSC003FailureMessageListsAllDrifts: + def test_five_divergences_produce_five_rows(self) -> None: + """SC-003: N seeded divergences -> N rows in the failure table.""" + mcp_controls = [Control(id=f"CTRL-{i:02d}", status="PASS") for i in range(5)] + harness_controls = [Control(id=f"CTRL-{i:02d}", status="FAIL") for i in range(5)] + report = compare(_mcp(*mcp_controls), _harness(*harness_controls), "sc003") + + assert len(report.disallowed_drifts) == 5 + + table = report.format_failure_table() + # Count table rows (excluding header + separator + preamble lines). + # A "data row" starts with "| CTRL-". + data_rows = [ln for ln in table.split("\n") if ln.startswith("| CTRL-")] + assert len(data_rows) == 5, f"Expected 5 rows, got {len(data_rows)}" + + +class TestLC1AllowedDriftResolutions: + """LC1: PENDING_LLM (MCP) -> non-PENDING_LLM (harness) is allowed for + every non-PENDING_LLM value the harness might land on.""" + + def test_pending_to_warn_is_allowed(self) -> None: + mcp = _mcp(Control(id="X", status="PENDING_LLM")) + harness = _harness(Control(id="X", status="WARN")) + report = compare(mcp, harness, "lc1") + assert report.is_green + assert len(report.drifts) == 1 + assert report.drifts[0].is_allowed_drift + + def test_pending_to_pass_is_allowed(self) -> None: + mcp = _mcp(Control(id="X", status="PENDING_LLM")) + harness = _harness(Control(id="X", status="PASS")) + report = compare(mcp, harness, "lc1") + assert report.is_green + assert len(report.drifts) == 1 + assert report.drifts[0].is_allowed_drift + + def test_pending_to_fail_is_allowed(self) -> None: + mcp = _mcp(Control(id="X", status="PENDING_LLM")) + harness = _harness(Control(id="X", status="FAIL")) + report = compare(mcp, harness, "lc1") + assert report.is_green + assert len(report.drifts) == 1 + assert report.drifts[0].is_allowed_drift + + +class TestDisallowedReverseDrift: + def test_reverse_drift_pending_llm_from_harness_disallowed(self) -> None: + """Harness produced PENDING_LLM while MCP resolved to WARN. That's + a bug: the harness's LLM continuation loop should resolve. Not + the other way around.""" + mcp = _mcp(Control(id="X", status="WARN")) + harness = _harness(Control(id="X", status="PENDING_LLM")) + report = compare(mcp, harness, "reverse") + assert not report.is_green + assert len(report.disallowed_drifts) == 1 + + +class TestMissingControlCase: + """T1-3: control appears on one side but not the other -> hard fail.""" + + def test_control_missing_on_harness_side_flagged(self) -> None: + mcp = _mcp( + Control(id="X", status="PASS"), + Control(id="Y", status="PASS"), + ) + harness = _harness(Control(id="X", status="PASS")) + report = compare(mcp, harness, "missing") + assert not report.is_green + assert any("missing on harness side" in d.note for d in report.disallowed_drifts) + + def test_control_missing_on_mcp_side_flagged(self) -> None: + mcp = _mcp(Control(id="X", status="PASS")) + harness = _harness( + Control(id="X", status="PASS"), + Control(id="Z", status="FAIL"), + ) + report = compare(mcp, harness, "missing") + assert not report.is_green + assert any("missing on mcp side" in d.note for d in report.disallowed_drifts) diff --git a/tests/darnit/parity/tier1/test_corpus_inventory.py b/tests/darnit/parity/tier1/test_corpus_inventory.py new file mode 100644 index 0000000..50ec04e --- /dev/null +++ b/tests/darnit/parity/tier1/test_corpus_inventory.py @@ -0,0 +1,47 @@ +"""Corpus inventory tests (feature 028 T015). + +Covers SC-008: the fixture corpus produces at least one control in each +of the four categories: all_pass, all_fail, mixed, pending_llm. + +Also guards against accidental fixture deletion (asserts at least four +fixtures are discovered). +""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from tests.darnit.parity.tier1.fixture_meta import ( + VALID_CATEGORIES, + load_parity_metadata, +) + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +def _discover_fixtures() -> list[Path]: + if not FIXTURES_DIR.exists(): + return [] + return sorted(p for p in FIXTURES_DIR.iterdir() if p.is_dir() and (p / ".baseline.toml").exists()) + + +class TestCorpusInventory: + def test_at_least_four_fixtures_present(self) -> None: + fixtures = _discover_fixtures() + assert len(fixtures) >= 4, ( + f"Expected at least 4 fixtures under {FIXTURES_DIR}, got {len(fixtures)}: {[f.name for f in fixtures]}" + ) + + def test_every_category_represented(self) -> None: + """SC-008: at least one fixture per category, verified via parity.toml.""" + fixtures = _discover_fixtures() + + categories: Counter[str] = Counter() + for fixture in fixtures: + meta = load_parity_metadata(fixture) + if meta is not None: + categories[meta.category] += 1 + + missing = [c for c in VALID_CATEGORIES if categories[c] < 1] + assert not missing, f"Categories missing from corpus: {missing}. Present: {dict(categories)}" diff --git a/tests/darnit/parity/tier1/test_fixture_meta.py b/tests/darnit/parity/tier1/test_fixture_meta.py new file mode 100644 index 0000000..5dd421b --- /dev/null +++ b/tests/darnit/parity/tier1/test_fixture_meta.py @@ -0,0 +1,134 @@ +"""Tests for parity.toml parser (feature 028 T007). + +Covers contract parity-toml-schema.md rules PT-2..PT-14. +""" + +from __future__ import annotations + +import pytest + +from tests.darnit.parity.tier1.fixture_meta import ( + ExpectedControl, + load_parity_metadata, +) + + +class TestBasicShape: + def test_absent_parity_toml_returns_none(self, tmp_path): + """PT-2: absent file -> None (not an error).""" + assert load_parity_metadata(tmp_path) is None + + def test_valid_minimal_parity_toml(self, tmp_path): + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "all_pass"\n', + ) + meta = load_parity_metadata(tmp_path) + assert meta is not None + assert meta.category == "all_pass" + assert meta.has_pending_llm is False # derived from empty counts + assert meta.strict is False + assert meta.counts == {} + + def test_valid_full_parity_toml(self, tmp_path): + (tmp_path / "parity.toml").write_text( + "[expected]\n" + 'category = "mixed"\n' + "has_pending_llm = true\n" + "strict = true\n" + "\n" + "[expected.counts]\n" + "pass = 4\n" + "fail = 2\n" + "warn = 1\n" + "error = 0\n" + "n_a = 3\n" + "pending_llm = 1\n" + "\n" + "[[expected.controls]]\n" + 'id = "OSPS-GV-01.01"\n' + 'status = "PASS"\n', + ) + meta = load_parity_metadata(tmp_path) + assert meta is not None + assert meta.category == "mixed" + assert meta.has_pending_llm is True + assert meta.strict is True + assert meta.counts == { + "pass": 4, + "fail": 2, + "warn": 1, + "error": 0, + "n_a": 3, + "pending_llm": 1, + } + assert meta.controls == (ExpectedControl(id="OSPS-GV-01.01", status="PASS"),) + + +class TestValidationFailures: + def test_malformed_toml_raises(self, tmp_path): + """PT-4: malformed TOML -> ValueError.""" + (tmp_path / "parity.toml").write_text("this is not valid toml [") + with pytest.raises(ValueError, match="malformed parity.toml"): + load_parity_metadata(tmp_path) + + def test_unknown_category_rejected(self, tmp_path): + """PT-5: category must be one of the four literals.""" + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "unknown"\n', + ) + with pytest.raises(ValueError, match="category.*must be one of"): + load_parity_metadata(tmp_path) + + def test_has_pending_llm_disagreement_rejected(self, tmp_path): + """PT-6: has_pending_llm=true with counts.pending_llm=0 fails.""" + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "mixed"\nhas_pending_llm = true\n\n[expected.counts]\npending_llm = 0\n', + ) + with pytest.raises(ValueError, match="disagrees with counts"): + load_parity_metadata(tmp_path) + + def test_negative_count_rejected(self, tmp_path): + """PT-8: counts must be non-negative.""" + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "mixed"\n[expected.counts]\npass = -1\n', + ) + with pytest.raises(ValueError, match="non-negative integer"): + load_parity_metadata(tmp_path) + + def test_missing_expected_section_rejected(self, tmp_path): + """A parity.toml without [expected] is a schema error.""" + (tmp_path / "parity.toml").write_text("# no expected block\n") + with pytest.raises(ValueError, match="missing.*expected"): + load_parity_metadata(tmp_path) + + +class TestForwardCompatibility: + def test_unknown_key_warns_but_does_not_fail(self, tmp_path): + """PT-9: unknown [expected] keys warn but don't fail.""" + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "all_pass"\nfuture_field = "someday"\n', + ) + with pytest.warns(UserWarning, match="unknown.*future_field"): + meta = load_parity_metadata(tmp_path) + assert meta is not None + assert meta.category == "all_pass" + + def test_unknown_counts_key_warns_but_does_not_fail(self, tmp_path): + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "mixed"\n[expected.counts]\npass = 1\nfuture_status = 5\n', + ) + with pytest.warns(UserWarning, match="unknown counts key.*future_status"): + meta = load_parity_metadata(tmp_path) + assert meta is not None + assert meta.counts == {"pass": 1} # unknown key skipped + + +class TestDerivedFields: + def test_has_pending_llm_derived_when_absent(self, tmp_path): + """When has_pending_llm is not set, derive from counts.pending_llm > 0.""" + (tmp_path / "parity.toml").write_text( + '[expected]\ncategory = "pending_llm"\n[expected.counts]\npending_llm = 2\n', + ) + meta = load_parity_metadata(tmp_path) + assert meta is not None + assert meta.has_pending_llm is True diff --git a/tests/darnit/parity/tier1/test_mcp_vs_harness.py b/tests/darnit/parity/tier1/test_mcp_vs_harness.py new file mode 100644 index 0000000..7c2cecf --- /dev/null +++ b/tests/darnit/parity/tier1/test_mcp_vs_harness.py @@ -0,0 +1,61 @@ +"""Tier 1 MCP-vs-harness parity test (feature 028 T013). + +Parametrized per fixture. For each fixture in the corpus, invokes both +audit paths and asserts they agree modulo the sole allowed drift class +(PENDING_LLM -> non-PENDING_LLM). Emits a summary line on every run +(FR-013) captured by capsys. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from tests.darnit.parity.tier1.comparator import AuditResult, compare +from tests.darnit.parity.tier1.fixture_meta import load_parity_metadata + + +def test_parity( + fixture_dir: Path, + mcp_tool_result: AuditResult, + harness_result: AuditResult, + capsys: pytest.CaptureFixture[str], +) -> None: + """Verify MCP tool and harness produce identical per-control status + for the given fixture, modulo the allowed PENDING_LLM -> non-PENDING_LLM + drift. + + Fixtures may declare `control_ids` in their `parity.toml` to filter + which controls are compared. Neither audit path applies fixture-level + `audit_profiles` automatically today, so this test-side filter is how + a fixture narrows its scope. + """ + meta = load_parity_metadata(fixture_dir) + if meta is not None and meta.control_ids: + mcp = mcp_tool_result.filter_to(meta.control_ids) + harness = harness_result.filter_to(meta.control_ids) + else: + mcp = mcp_tool_result + harness = harness_result + + report = compare(mcp, harness, fixture_name=fixture_dir.name) + + # FR-013 evidence: emit summary line to stdout unconditionally, so + # even green runs produce a per-fixture record. + summary = report.format_summary_line() + print(summary) + + # MC2 fix: assert the summary line matches the FR-013 pattern. + pattern = re.compile( + rf"^\[tier1\] {re.escape(fixture_dir.name)}: " + r"\d+ controls compared, \d+ agreed, \d+ diverged, \d+ allowed-drift$", + ) + assert pattern.match(summary), f"FR-013 evidence line malformed: {summary!r}" + + if not report.is_green: + # Emit the drift table into stdout as well for CI logs. + table = report.format_failure_table() + print(table) + pytest.fail("MCP tool and harness disagree beyond allowed drift:\n" + table) diff --git a/tests/darnit/parity/tier1/test_no_product_changes.py b/tests/darnit/parity/tier1/test_no_product_changes.py new file mode 100644 index 0000000..0b02b0e --- /dev/null +++ b/tests/darnit/parity/tier1/test_no_product_changes.py @@ -0,0 +1,131 @@ +"""FR-014 mechanical enforcement (feature 028 T031a). + +Runs `git diff --name-only ...HEAD` and asserts no file under +`packages/darnit/src/` or `packages/darnit-baseline/src/` is modified in +the current PR. Guards against a future maintainer accidentally adding +a "small helper" to product code from a parity-tests PR. + +Fails loudly (not silently) when the base ref cannot be reached under +CI, since a shallow-clone runner that silently passes gives the +maintainer a false sense of coverage. PR #370 review fix. +""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +_BASE_CANDIDATES = ( + # `upstream/main` before `origin/main`: on a fork clone, `origin/main` + # is often stale (last synced days ago) while `upstream/main` tracks + # the source-of-truth remote. On the source repo itself, + # `upstream/main` won't exist and we fall through to `origin/main`. + "upstream/main", + "origin/main", + "main", +) + + +def _find_reachable_base() -> str | None: + for candidate in _BASE_CANDIDATES: + rc = subprocess.run( + ["git", "rev-parse", "--verify", "-q", candidate], + capture_output=True, + text=True, + ) + if rc.returncode == 0 and rc.stdout.strip(): + return candidate + return None + + +def _base_ref() -> str | None: + """Detect the base ref to diff against. Returns None if none reachable + even after unshallowing. + + Tries the immediate stack parent first (026-harness-with-stage1) because + feature 028 is stacked on it during development; when 028 is on main + (after 026 merges), origin/main is the right base. The precise order + matters because a stacked-branch check against `main` would incorrectly + flag every 026 change as a violation. Under a shallow CI clone (the + default `actions/checkout` config) the base ref is often not initially + reachable; try `git fetch --unshallow` + a full remote sync once before + giving up. + """ + hit = _find_reachable_base() + if hit is not None: + return hit + + # Shallow-clone recovery path. `--unshallow` deepens the current ref, + # but stack-parent refs (`origin/026-harness-with-stage1`) live on + # OTHER branches -- add an explicit refspec fetch so they appear. + # Both commands quietly succeed even when there's no remote to fetch + # from (or the repo is already unshallow); either way we re-check. + subprocess.run( + ["git", "fetch", "--unshallow", "--tags", "origin"], + capture_output=True, + text=True, + ) + subprocess.run( + ["git", "fetch", "origin", "+refs/heads/*:refs/remotes/origin/*"], + capture_output=True, + text=True, + ) + return _find_reachable_base() + + +def _is_ci() -> bool: + """True on GitHub Actions and most other CI runners. + + The CI env var is set by GitHub Actions, GitLab CI, CircleCI, + Travis, and Buildkite; that's a good-enough shibboleth for + turning the "no base ref" case from skip -> fail. + """ + return bool(os.environ.get("CI")) + + +def test_no_product_source_changes() -> None: + """FR-014: parity-tests PR MUST NOT modify product source. Test/config + files are exempt so a build-config touch or a test refactor stays in + scope. + + Now that PR #365 has merged, feature 028 sits directly on `main`; + the base ref is `origin/main` (or a local `main` variant) and the + diff cleanly identifies the parity PR's own commits. If no main ref + is reachable at all -- e.g., a shallow CI clone without unshallow + -- fall back to skip (local dev) or fail (CI, with a clear pointer + at fetch depth). + """ + base = _base_ref() + if base is None: + # PR #370 review fix: silently skipping under shallow CI clones + # gave a false sense of coverage. Skip only when running locally; + # under CI, fail with a clear pointer at the fix (fetch depth). + if _is_ci(): + pytest.fail( + "FR-014 check cannot run: no base ref reachable and CI=1. " + "Configure the workflow to fetch enough history (e.g., " + "actions/checkout with fetch-depth: 0) so this check can " + "compare against `origin/main`.", + ) + pytest.skip("no base ref reachable (local dev); CI enforces this check") + + rc = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD"], + capture_output=True, + text=True, + check=True, + ) + changed = [ln.strip() for ln in rc.stdout.splitlines() if ln.strip()] + + forbidden = [ + f for f in changed if (f.startswith("packages/darnit/src/") or f.startswith("packages/darnit-baseline/src/")) + ] + assert not forbidden, ( + "Feature 028 (parity tests) MUST NOT modify product source code " + "(FR-014). Offending files:\n" + + "\n".join(f" - {f}" for f in forbidden) + + "\n\nIf a change under packages/*/src/ is genuinely necessary, " + "split it into a separate PR that is not scoped to feature 028." + ) diff --git a/tests/darnit/parity/tier2/__init__.py b/tests/darnit/parity/tier2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/darnit/parity/tier2/artifact_writer.py b/tests/darnit/parity/tier2/artifact_writer.py new file mode 100644 index 0000000..ab8c1d7 --- /dev/null +++ b/tests/darnit/parity/tier2/artifact_writer.py @@ -0,0 +1,46 @@ +"""Artifact bundle writer for Tier 2 (feature 028 T019). + +Writes per-fixture artifacts under parity-artifacts// for +inspection after a workflow run. Even on GREEN runs the bundle is written +so a maintainer can verify what the test saw. See data-model.md section 7. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + + +def write_fixture_artifacts( + artifact_root: Path, + fixture_name: str, + mcp_json: str, + skill_markdown: str, + diff_md: str, + metadata: dict[str, object] | None = None, +) -> Path: + """Write the four per-fixture artifact files. Returns the fixture's + artifact directory. + """ + fixture_dir = artifact_root / fixture_name + fixture_dir.mkdir(parents=True, exist_ok=True) + + (fixture_dir / "mcp_tool_result.json").write_text(mcp_json) + (fixture_dir / "skill_final_message.md").write_text(skill_markdown) + (fixture_dir / "diff_report.md").write_text(diff_md) + + meta_out: dict[str, object] = { + "fixture_name": fixture_name, + "timestamp_utc": datetime.now(UTC).isoformat(), + } + if metadata: + meta_out.update(metadata) + (fixture_dir / "metadata.json").write_text( + json.dumps(meta_out, indent=2, default=str), + ) + + return fixture_dir + + +__all__ = ("write_fixture_artifacts",) diff --git a/tests/darnit/parity/tier2/claude_agent_sdk_client.py b/tests/darnit/parity/tier2/claude_agent_sdk_client.py new file mode 100644 index 0000000..ee474ba --- /dev/null +++ b/tests/darnit/parity/tier2/claude_agent_sdk_client.py @@ -0,0 +1,143 @@ +"""Thin wrapper around `claude_agent_sdk.query` for Tier 2 (T018). + +Invokes the /darnit-audit skill against a fixture and returns the final +assistant message. Deterministic invocation: no streaming interrupts, no +interactive turns; explicit prompt + model + max_turns. + +Per research.md R7: + - Prompt content is loaded from skill_prompt_snapshot.md. + - Model defaults to anthropic:claude-sonnet-5 (matches feature 025/026). + - max_turns cap prevents runaway budget consumption. + - ANTHROPIC_API_KEY MUST be set (FR-010); absence raises SetupError. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +class SetupError(RuntimeError): + """Raised when Tier 2 lacks the environment it needs to invoke the skill.""" + + +@dataclass(frozen=True) +class SkillInvocationResult: + """Return type for `invoke_skill`.""" + + final_message: str + model: str + turn_count: int + metadata: dict[str, object] + + +PROMPT_SNAPSHOT_PATH = Path(__file__).parent / "skill_prompt_snapshot.md" + + +def _load_skill_prompt() -> str: + if not PROMPT_SNAPSHOT_PATH.exists(): + raise SetupError( + f"skill prompt snapshot missing: {PROMPT_SNAPSHOT_PATH}. Run T022 to capture it before invoking Tier 2.", + ) + return PROMPT_SNAPSHOT_PATH.read_text() + + +def _check_env() -> None: + """FR-010: fail fast when ANTHROPIC_API_KEY is absent.""" + if not os.environ.get("ANTHROPIC_API_KEY"): + raise SetupError( + "Tier 2 requires ANTHROPIC_API_KEY. Configure the " + "`parity-tier2` GitHub Actions Environment (or export the var " + "for a local run).", + ) + + +async def invoke_skill( + fixture_dir: Path, + model: str = "anthropic:claude-sonnet-5", + max_turns: int = 20, +) -> SkillInvocationResult: + """Invoke the /darnit-audit skill against `fixture_dir`. + + Returns a `SkillInvocationResult`. Raises SetupError on missing env + (before any API call is made). + """ + _check_env() + skill_prompt = _load_skill_prompt() + + # Import here so unit tests exercising the SetupError path don't + # require the SDK to be importable. + from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ResultMessage, + TextBlock, + query, + ) + + user_prompt = ( + f"{skill_prompt}\n\n" + f"Run the audit against the repository at {fixture_dir}. " + f"Summarize the results per the skill's usual format." + ) + + # PR #370 review fix: wire the darnit MCP server, allow the audit + # tool, and lock the agent down so it can't reach for out-of-band + # settings. Previously ClaudeAgentOptions passed only model, cwd, + # and max_turns -- the agent had NO way to call + # audit_openssf_baseline and the parity test measured "skill does + # nothing" instead of "skill vs tool". + options = ClaudeAgentOptions( + model=model.replace("anthropic:", ""), # SDK expects bare model name + max_turns=max_turns, + cwd=str(fixture_dir), + mcp_servers={ + "darnit": { + "type": "stdio", + "command": "darnit", + "args": ["serve", "--framework", "openssf-baseline"], + }, + }, + # Restrict to the specific MCP tool the parity test needs. Any + # off-list tool call is rejected by the SDK. + allowed_tools=["mcp__darnit__audit_openssf_baseline"], + # Auto-accept the tool call; parity CI is non-interactive. + permission_mode="acceptEdits", + # Isolate from the running host's Claude Code settings so a + # local dev's project/user config can't influence the run. + setting_sources=[], + ) + + final_text: str = "" + turn_count = 0 + result_meta: dict[str, object] = {} + + async for message in query(prompt=user_prompt, options=options): + if isinstance(message, AssistantMessage): + turn_count += 1 + # Concatenate any text blocks from this assistant turn. + for block in message.content: + if isinstance(block, TextBlock): + final_text = block.text # keep only the LAST turn's text + elif isinstance(message, ResultMessage): + # ResultMessage carries final metadata (usage, cost, etc.) + result_meta = { + "duration_ms": getattr(message, "duration_ms", None), + "num_turns": getattr(message, "num_turns", turn_count), + } + + return SkillInvocationResult( + final_message=final_text, + model=model, + turn_count=turn_count, + metadata=result_meta, + ) + + +__all__ = ( + "SetupError", + "SkillInvocationResult", + "invoke_skill", + "PROMPT_SNAPSHOT_PATH", +) diff --git a/tests/darnit/parity/tier2/diff.py b/tests/darnit/parity/tier2/diff.py new file mode 100644 index 0000000..822fe71 --- /dev/null +++ b/tests/darnit/parity/tier2/diff.py @@ -0,0 +1,182 @@ +"""Tier 2 diff: MCP tool JSON vs skill Markdown summary (feature 028 T020). + +Compares the skill's user-facing summary against the raw tool output. +Any per-control status difference is a hard failure regardless of +authority level (FR-008 / T2-13). Unparseable skill output is a distinct +failure class from disagreement (FR-006a). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum + +from tests.darnit.parity.tier1.comparator import AuditResult +from tests.darnit.parity.tier2.skill_markdown_parser import SkillReport + + +class Tier2Outcome(IntEnum): + """Contract T2-13 exit codes.""" + + SUCCESS = 0 + PER_CONTROL_DISAGREE = 1 + SKILL_UNPARSEABLE = 2 + COUNTS_DISAGREE = 1 # combined with per-control under exit 1 + + +@dataclass(frozen=True) +class Tier2DiffReport: + fixture_name: str + outcome: str # "success" | "per_control_disagree" | "skill_unparseable" | "counts_disagree" + disagreeing_controls: tuple[str, ...] = () + diff_markdown: str = "" + + @property + def is_success(self) -> bool: + return self.outcome == "success" + + +def diff( + mcp_result: AuditResult, + skill_report: SkillReport, + fixture_name: str, +) -> Tier2DiffReport: + """Compare tool JSON vs skill summary. + + Order of classification (most severe first): + 1. Skill output unparseable -> SKILL_UNPARSEABLE. + 2. Per-control status disagreement -> PER_CONTROL_DISAGREE. + 3. Summary-count disagreement (per-control agrees) -> COUNTS_DISAGREE. + 4. All-agree -> success. + """ + if not skill_report.parseable: + return Tier2DiffReport( + fixture_name=fixture_name, + outcome="skill_unparseable", + diff_markdown=_format_unparseable_report(fixture_name, skill_report), + ) + + # Per-control comparison: iterate both directions so a skill that + # OMITS a control is caught. Previously only skill claims were + # walked, so silently dropping a FAIL was reported as "success". + # PR #370 review fix. + mcp_by_id = {c.id: c for c in mcp_result.controls} + disagreements: list[tuple[str, str, str]] = [] # (control_id, tool_status, skill_status) + + assert skill_report.controls is not None + skill_by_id = {claim.id: claim for claim in skill_report.controls} + for claim in skill_report.controls: + tool_ctrl = mcp_by_id.get(claim.id) + if tool_ctrl is None: + disagreements.append((claim.id, "", claim.status)) + continue + if tool_ctrl.status != claim.status: + disagreements.append((claim.id, tool_ctrl.status, claim.status)) + + # Tool controls the skill did not mention at all. + for tool_ctrl in mcp_result.controls: + if tool_ctrl.id not in skill_by_id: + disagreements.append( + (tool_ctrl.id, tool_ctrl.status, ""), + ) + + if disagreements: + return Tier2DiffReport( + fixture_name=fixture_name, + outcome="per_control_disagree", + disagreeing_controls=tuple(d[0] for d in disagreements), + diff_markdown=_format_per_control_report(fixture_name, disagreements), + ) + + # Counts comparison. Compute tool counts from AuditResult. + assert skill_report.counts is not None + tool_counts = _tool_counts(mcp_result) + + counts_differ = False + for key in ("pass", "fail", "warn", "error"): + if key in skill_report.counts and skill_report.counts[key] != tool_counts.get(key, 0): + counts_differ = True + + if counts_differ: + return Tier2DiffReport( + fixture_name=fixture_name, + outcome="counts_disagree", + diff_markdown=_format_counts_report( + fixture_name, + tool_counts, + skill_report.counts, + ), + ) + + return Tier2DiffReport( + fixture_name=fixture_name, + outcome="success", + diff_markdown=f"# Tier 2 parity: {fixture_name}\n\nSUCCESS: skill agrees with tool.\n", + ) + + +def _tool_counts(mcp_result: AuditResult) -> dict[str, int]: + counts = {"pass": 0, "fail": 0, "warn": 0, "error": 0, "n_a": 0, "pending_llm": 0} + for c in mcp_result.controls: + key = c.status.lower().replace("/", "_") + if key in counts: + counts[key] += 1 + return counts + + +def _format_per_control_report( + fixture_name: str, + disagreements: list[tuple[str, str, str]], +) -> str: + lines = [ + f"# Tier 2 parity: {fixture_name}", + "", + "FAIL: per-control status disagreement between skill summary and tool JSON.", + "", + "| control_id | tool_status | skill_status |", + "| --- | --- | --- |", + ] + for cid, tool_s, skill_s in disagreements: + lines.append(f"| {cid} | {tool_s} | {skill_s} |") + lines.append("") + lines.append( + "See `mcp_tool_result.json` and `skill_final_message.md` in this directory for the raw artifacts.", + ) + return "\n".join(lines) + + +def _format_counts_report( + fixture_name: str, + tool_counts: dict[str, int], + skill_counts: dict[str, int], +) -> str: + lines = [ + f"# Tier 2 parity: {fixture_name}", + "", + "FAIL: summary counts differ between skill summary and tool JSON.", + "", + "| status | tool_count | skill_count |", + "| --- | --- | --- |", + ] + for key in ("pass", "fail", "warn", "error", "n_a", "pending_llm"): + t = tool_counts.get(key, 0) + s = skill_counts.get(key, "-") + lines.append(f"| {key} | {t} | {s} |") + return "\n".join(lines) + + +def _format_unparseable_report( + fixture_name: str, + skill_report: SkillReport, +) -> str: + return ( + f"# Tier 2 parity: {fixture_name}\n\n" + "FAIL: skill output could not be parsed. This is DISTINCT from a " + "disagreement -- the parser did not find the expected shape in the " + "skill's final message.\n\n" + f"Parser notes: {list(skill_report.parse_notes) or ['(none)']}\n\n" + "See `skill_final_message.md` in this directory for the raw output.\n" + ) + + +__all__ = ("Tier2Outcome", "Tier2DiffReport", "diff") diff --git a/tests/darnit/parity/tier2/golden/all_pass.md b/tests/darnit/parity/tier2/golden/all_pass.md new file mode 100644 index 0000000..e8fd88a --- /dev/null +++ b/tests/darnit/parity/tier2/golden/all_pass.md @@ -0,0 +1,17 @@ +# Darnit Audit Report + +## Summary + +- Total: 2 +- Passed: 2 +- Failed: 0 +- Warned: 0 + +## Passed Controls + +- OSPS-DO-01.01 PASS (dispositive) -- README file present +- OSPS-LE-03.01 PASS (dispositive) -- LICENSE file present + +## Failed Controls + +None. diff --git a/tests/darnit/parity/tier2/golden/mixed_drift.md b/tests/darnit/parity/tier2/golden/mixed_drift.md new file mode 100644 index 0000000..1e236d9 --- /dev/null +++ b/tests/darnit/parity/tier2/golden/mixed_drift.md @@ -0,0 +1,19 @@ +# Audit Summary + +I ran the darnit audit against this repository. Here's what I found: + +**Summary**: 51 PASS, 5 FAIL, 7 WARN, 2 ERROR out of 66 controls. + +## Failed Controls + +- **OSPS-BR-06.01**: FAIL -- no signed releases found +- **OSPS-VM-04.01**: FAIL -- SECURITY.md not present + +## Warned Controls + +- **OSPS-GV-03.01**: WARN -- some ambiguity in the CODEOWNERS file +- **OSPS-DO-04.01**: WARN -- documentation could be improved + +## Suggestions + +Consider adding a SECURITY.md file to address OSPS-VM-04.01. diff --git a/tests/darnit/parity/tier2/golden/unparseable.md b/tests/darnit/parity/tier2/golden/unparseable.md new file mode 100644 index 0000000..084c5bf --- /dev/null +++ b/tests/darnit/parity/tier2/golden/unparseable.md @@ -0,0 +1 @@ +Sorry, I encountered an issue and could not complete the audit. Please try again. diff --git a/tests/darnit/parity/tier2/run.py b/tests/darnit/parity/tier2/run.py new file mode 100644 index 0000000..549a35f --- /dev/null +++ b/tests/darnit/parity/tier2/run.py @@ -0,0 +1,207 @@ +"""Tier 2 runner entrypoint (feature 028 T021). + +Invoked from the manual-dispatch GitHub Actions workflow. For each fixture: + + 1. Capture MCP tool JSON via direct Python call. + 2. Invoke the /darnit-audit skill via the Claude Agent SDK. + 3. Parse the skill's final assistant message. + 4. Run `diff()` and write per-fixture artifacts. + 5. Aggregate outcomes into an exit code per contract T2-13. + +Exit codes: + 0 -- success (every fixture agrees) + 1 -- at least one fixture had a skill-vs-tool disagreement + 2 -- at least one fixture had unparseable skill output + 3 -- setup error (missing key, missing fixtures) + 4 -- rate limit exhausted (not automated; documented for follow-up) + +`--dry-run` stubs the SDK client with a canned response so the runner +can be exercised offline (used by the config workflow test). +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path + +from tests.darnit.parity.tier1.comparator import AuditResult +from tests.darnit.parity.tier2.artifact_writer import write_fixture_artifacts +from tests.darnit.parity.tier2.claude_agent_sdk_client import ( + SetupError, + SkillInvocationResult, +) +from tests.darnit.parity.tier2.diff import diff +from tests.darnit.parity.tier2.skill_markdown_parser import SkillReport + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" +DEFAULT_ARTIFACT_ROOT = Path("parity-artifacts") + + +def _discover_fixtures(fixture_glob: str) -> list[Path]: + if not FIXTURES_DIR.exists(): + return [] + all_fixtures = sorted(p for p in FIXTURES_DIR.iterdir() if p.is_dir() and (p / ".baseline.toml").exists()) + if fixture_glob == "*": + return all_fixtures + import fnmatch + + return [p for p in all_fixtures if fnmatch.fnmatch(p.name, fixture_glob)] + + +def _run_mcp_tool(fixture_dir: Path) -> tuple[str, AuditResult]: + """Invoke the MCP tool; return (raw_json_str, normalized_result).""" + from darnit_baseline.tools import audit_openssf_baseline + + raw = audit_openssf_baseline( + local_path=str(fixture_dir), + level=3, + output_format="json", + auto_init_config=False, + attest=False, + prefer_upstream=False, + ) + return raw, AuditResult.from_mcp_json(json.loads(raw)) + + +async def _run_skill( + fixture_dir: Path, + dry_run: bool, +) -> SkillInvocationResult: + if dry_run: + # Stub: return a canned Markdown summary that the parser can extract. + return SkillInvocationResult( + final_message=("# Dry-Run Skill Output\n\nPassed: 0\nFailed: 0\nWarned: 0\n\nNo controls to summarize."), + model="dry-run", + turn_count=0, + metadata={"dry_run": True}, + ) + from tests.darnit.parity.tier2.claude_agent_sdk_client import invoke_skill + + return await invoke_skill(fixture_dir=fixture_dir) + + +def _write_step_summary(text: str) -> None: + """Append `text` to GITHUB_STEP_SUMMARY if set. No-op otherwise.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + with open(path, "a") as f: + f.write(text + "\n") + + +def _preflight_summary(fixture_glob: str) -> None: + """T2-7/T2-8: preflight audit line BEFORE consuming the API key.""" + actor = os.environ.get("GITHUB_ACTOR", "") + sha = os.environ.get("GITHUB_SHA", "") + line = f"Tier 2 parity preflight: actor={actor} sha={sha} fixture_glob={fixture_glob!r}" + print(line, file=sys.stderr) + _write_step_summary(line) + + +async def _run_one_fixture( + fixture_dir: Path, + artifact_root: Path, + dry_run: bool, +) -> str: + """Return the diff outcome string for this fixture.""" + mcp_raw, mcp_result = _run_mcp_tool(fixture_dir) + skill_result = await _run_skill(fixture_dir, dry_run=dry_run) + skill_report = SkillReport.parse(skill_result.final_message) + diff_report = diff(mcp_result, skill_report, fixture_dir.name) + + write_fixture_artifacts( + artifact_root=artifact_root, + fixture_name=fixture_dir.name, + mcp_json=mcp_raw, + skill_markdown=skill_result.final_message, + diff_md=diff_report.diff_markdown, + metadata={ + "model": skill_result.model, + "turn_count": skill_result.turn_count, + "dry_run": dry_run, + }, + ) + return diff_report.outcome + + +async def _main_async(args: argparse.Namespace) -> int: + _preflight_summary(args.fixture_glob) + + artifact_root = Path(args.artifact_dir) + fixtures = _discover_fixtures(args.fixture_glob) + if not fixtures: + print( + f"Tier 2: no fixtures matched {args.fixture_glob!r} under {FIXTURES_DIR}", + file=sys.stderr, + ) + return 3 # setup error + + outcomes: dict[str, list[str]] = { + "success": [], + "per_control_disagree": [], + "counts_disagree": [], + "skill_unparseable": [], + } + + for fixture_dir in fixtures: + try: + outcome = await _run_one_fixture(fixture_dir, artifact_root, args.dry_run) + except SetupError as exc: + print(f"Tier 2 setup error: {exc}", file=sys.stderr) + _write_step_summary(f"setup_error: {exc}") + return 3 + except Exception as exc: # noqa: BLE001 + print( + f"Tier 2 fixture {fixture_dir.name!r} raised {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + outcomes.setdefault("errored", []).append(fixture_dir.name) + continue + outcomes[outcome].append(fixture_dir.name) + + summary = ( + f"Tier 2 parity check: {len(fixtures)} fixtures checked, " + f"{len(outcomes['per_control_disagree']) + len(outcomes['counts_disagree'])} drifts, " + f"{len(outcomes['skill_unparseable'])} unparseable" + ) + print(summary, file=sys.stderr) + _write_step_summary(summary) + + # Compute exit code per T2-13. + if outcomes.get("errored"): + return 3 + if outcomes["per_control_disagree"] or outcomes["counts_disagree"]: + return 1 + if outcomes["skill_unparseable"]: + return 2 + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Feature 028 Tier 2 parity runner") + parser.add_argument( + "--fixture-glob", + default="*", + help="Glob to filter which fixtures under tests/darnit/parity/fixtures/ are run", + ) + parser.add_argument( + "--artifact-dir", + default=str(DEFAULT_ARTIFACT_ROOT), + help="Where to write per-fixture artifact bundles", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Stub the SDK invocation with a canned response (no API call)", + ) + args = parser.parse_args(argv) + + return asyncio.new_event_loop().run_until_complete(_main_async(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/darnit/parity/tier2/skill_markdown_parser.py b/tests/darnit/parity/tier2/skill_markdown_parser.py new file mode 100644 index 0000000..8b62f30 --- /dev/null +++ b/tests/darnit/parity/tier2/skill_markdown_parser.py @@ -0,0 +1,182 @@ +"""Best-effort parser for the /darnit-audit skill's final assistant message. + +Feature 028 T016. The skill's output format is NOT a stable contract; this +parser is heuristic. A parse failure surfaces as `parseable=False` (a +distinct failure class from "skill and tool disagree"), never a crash. + +See: + - specs/028-audit-parity-tests/data-model.md section 6 + - specs/028-audit-parity-tests/research.md R6 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + + +@dataclass(frozen=True) +class SkillControlClaim: + id: str + status: Literal["PASS", "FAIL", "WARN", "N/A", "ERROR", "PENDING_LLM"] + + +@dataclass(frozen=True) +class SkillReport: + parseable: bool + raw_markdown: str + counts: dict[str, int] | None = None + controls: tuple[SkillControlClaim, ...] | None = None + parse_notes: tuple[str, ...] = () + + @classmethod + def parse(cls, markdown: str) -> SkillReport: + """Best-effort regex parser. Never raises.""" + try: + counts = _extract_counts(markdown) + controls = _extract_controls(markdown) + notes: list[str] = [] + + if counts is None: + notes.append("could not extract summary counts") + if controls is None or not controls: + notes.append("could not extract per-control claims") + + parseable = counts is not None and controls is not None and len(controls) > 0 + return cls( + parseable=parseable, + raw_markdown=markdown, + counts=counts, + controls=tuple(controls) if controls else None, + parse_notes=tuple(notes), + ) + except Exception as exc: # noqa: BLE001 + return cls( + parseable=False, + raw_markdown=markdown, + counts=None, + controls=None, + parse_notes=(f"parser exception: {type(exc).__name__}: {exc}",), + ) + + +_STATUS_LITERALS = ("PASS", "FAIL", "WARN", "N/A", "ERROR", "PENDING_LLM") + + +def _extract_counts(markdown: str) -> dict[str, int] | None: + """Extract summary counts. Recognizes shapes like: + - "Passed: 51", "Failed: 5", "Warned: 7" (verbose "- Passed: N" lines) + - "PASS: 51, FAIL: 5" (colon form) + - "51 PASS, 5 FAIL, 7 WARN" (adjacency form -- LAST because it can + false-match a per-control line like "OSPS-DO-01.01 PASS") + """ + counts: dict[str, int] = {} + + # Shape 3 (verbose) FIRST -- most specific, least prone to false matches. + for verbose, canonical in [ + ("passed", "pass"), + ("failed", "fail"), + ("warned", "warn"), + ("errored", "error"), + ("na", "n_a"), + ]: + pattern = re.compile(rf"\b{verbose}\s*:\s*(\d+)\b", re.IGNORECASE) + m = pattern.search(markdown) + if m: + counts[canonical] = int(m.group(1)) + + # Shape 2: "STATUS: N" (still specific because of the colon). + for status in _STATUS_LITERALS: + key = status.lower().replace("/", "_") + if key in counts: + continue + pattern = re.compile( + rf"\b{re.escape(status)}\s*:\s*(\d+)\b", + re.IGNORECASE, + ) + m = pattern.search(markdown) + if m: + counts[key] = int(m.group(1)) + + # Shape 1: "N STATUS" or "N/M STATUS" -- least specific, most prone to + # false-match a per-control line. Only apply if the count wasn't found + # via the more specific shapes above. + for status in _STATUS_LITERALS: + key = status.lower().replace("/", "_") + if key in counts: + continue + # Prefer patterns with N/M or N followed by "STATUS," or the word + # is followed by a comma / end-of-line to reduce false matches on + # per-control lines that have "01 PASS (dispositive)" shapes. + pattern = re.compile( + rf"\b(\d+)/\d+\s+{re.escape(status)}\b", + re.IGNORECASE, + ) + m = pattern.search(markdown) + if m: + counts[key] = int(m.group(1)) + continue + + pattern = re.compile( + rf"(? list[SkillControlClaim] | None: + """Extract per-control claims. Recognizes shapes like: + - "**OSPS-GV-01.01**: PASS" + - "- OSPS-GV-01.01 PASS" + - "OSPS-GV-01.01: PASS" + Returns None on total parse failure; empty list is a valid outcome when + the skill's summary omits per-control detail. + """ + control_id_pattern = r"(OSPS-[A-Z]{2}-\d{2}\.\d{2}|STAGE1-REF-[A-Z-]+-\d{2})" + claims: list[SkillControlClaim] = [] + seen: set[str] = set() + + for line in markdown.splitlines(): + # Try to find a control ID + status on the same line. + cid_match = re.search(control_id_pattern, line) + if not cid_match: + continue + + # PR #370 review fix: pick the LEFTMOST status literal on the + # line -- previously the loop matched by _STATUS_LITERALS order, + # so a phrase like "WARN ... to reach PASS" got read as PASS. + # PENDING_LLM contains "PENDING", so exclude any match whose + # start position is a substring of a longer literal that starts + # earlier on the line. + earliest_status: str | None = None + earliest_pos = len(line) + 1 + for status in _STATUS_LITERALS: + m = re.search(rf"\b{re.escape(status)}\b", line) + if m is None: + continue + if m.start() < earliest_pos or ( + m.start() == earliest_pos and len(status) > len(earliest_status or "") + ): + earliest_pos = m.start() + earliest_status = status + + if earliest_status is None: + continue + + cid = cid_match.group(1) + if cid in seen: + continue + seen.add(cid) + claims.append(SkillControlClaim(id=cid, status=earliest_status)) + + return claims + + +__all__ = ("SkillControlClaim", "SkillReport") diff --git a/tests/darnit/parity/tier2/skill_prompt_snapshot.md b/tests/darnit/parity/tier2/skill_prompt_snapshot.md new file mode 100644 index 0000000..296de7d --- /dev/null +++ b/tests/darnit/parity/tier2/skill_prompt_snapshot.md @@ -0,0 +1,32 @@ + + +You are the `/darnit-audit` skill. When invoked, run the darnit audit +against the current repository via the `audit_openssf_baseline` MCP tool +(or its equivalent), then produce a Markdown summary report. + +Your summary should include: + +1. A top-line count (Passed / Failed / Warned / N/A). +2. A list of failed controls with a brief remediation note per control. +3. A list of warned or pending controls with a brief explanation. +4. Optional: proactive suggestions the operator could take. + +Report every control the audit produced; do not silently reclassify +statuses. If the audit returns WARN for a control, your summary must +report WARN, not PASS. If the audit returns FAIL, your summary must +report FAIL. Presenting a status different from what the audit produced +is a governance violation this test suite is designed to detect. diff --git a/tests/darnit/parity/tier2/test_diff_adversarial.py b/tests/darnit/parity/tier2/test_diff_adversarial.py new file mode 100644 index 0000000..76c6f65 --- /dev/null +++ b/tests/darnit/parity/tier2/test_diff_adversarial.py @@ -0,0 +1,166 @@ +"""Adversarial diff tests (feature 028 T025). + +Cover SC-004, FR-006a, FR-008, FR-010. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.darnit.parity.tier1.comparator import AuditResult, Control +from tests.darnit.parity.tier2.diff import diff +from tests.darnit.parity.tier2.skill_markdown_parser import ( + SkillControlClaim, + SkillReport, +) + + +def _mcp(*controls: Control) -> AuditResult: + return AuditResult(controls=tuple(controls), source="mcp_tool") + + +def _parseable_skill( + *claims: SkillControlClaim, + counts: dict[str, int] | None = None, +) -> SkillReport: + return SkillReport( + parseable=True, + raw_markdown="stub", + counts=counts or {"pass": 0, "fail": 0, "warn": 0}, + controls=tuple(claims), + ) + + +class TestSC004SkillReclassificationCaught: + def test_warn_control_reclassified_as_pass_is_caught(self) -> None: + """SC-004: skill says PASS, tool says WARN -> per_control_disagree.""" + mcp = _mcp(Control(id="X", status="WARN", authority="suggestive")) + skill = _parseable_skill(SkillControlClaim(id="X", status="PASS")) + report = diff(mcp, skill, "sc004") + + assert report.outcome == "per_control_disagree" + assert "X" in report.disagreeing_controls + + def test_suggestive_authority_no_license_to_reinterpret(self) -> None: + """FR-008: even for suggestive-authority controls, the skill has + NO license to reinterpret the tool's verdict.""" + mcp = _mcp(Control(id="X", status="WARN", authority="suggestive")) + skill = _parseable_skill(SkillControlClaim(id="X", status="PASS")) + report = diff(mcp, skill, "auth") + assert report.outcome == "per_control_disagree" + + def test_dispositive_authority_also_caught(self) -> None: + mcp = _mcp(Control(id="Y", status="FAIL", authority="dispositive")) + skill = _parseable_skill(SkillControlClaim(id="Y", status="PASS")) + report = diff(mcp, skill, "auth-disp") + assert report.outcome == "per_control_disagree" + + +class TestFR006AUnparseableSkillOutput: + def test_unparseable_report_produces_distinct_outcome(self) -> None: + """FR-006a: unparseable skill output is a DISTINCT failure class, + NOT lumped in with 'skill and tool disagree'.""" + mcp = _mcp(Control(id="X", status="PASS")) + skill = SkillReport( + parseable=False, + raw_markdown="Sorry, I couldn't complete the audit.", + counts=None, + controls=None, + parse_notes=("could not extract summary counts",), + ) + report = diff(mcp, skill, "unparse") + assert report.outcome == "skill_unparseable" + assert "distinct" in report.diff_markdown.lower() or "unparseable" in report.diff_markdown.lower() + + +class TestCountsOnlyDisagreement: + def test_summary_counts_differ_but_per_control_agrees(self) -> None: + """Counts-only disagreement is its own outcome (weakest signal).""" + mcp = _mcp( + Control(id="A", status="PASS"), + Control(id="B", status="PASS"), + ) + skill = _parseable_skill( + SkillControlClaim(id="A", status="PASS"), + SkillControlClaim(id="B", status="PASS"), + counts={"pass": 99, "fail": 0, "warn": 0}, # tool has 2; skill claims 99 + ) + report = diff(mcp, skill, "counts") + assert report.outcome == "counts_disagree" + + +class TestSuccess: + def test_full_agreement_is_success(self) -> None: + mcp = _mcp( + Control(id="A", status="PASS"), + Control(id="B", status="FAIL"), + ) + skill = _parseable_skill( + SkillControlClaim(id="A", status="PASS"), + SkillControlClaim(id="B", status="FAIL"), + counts={"pass": 1, "fail": 1, "warn": 0}, + ) + report = diff(mcp, skill, "green") + assert report.outcome == "success" + + +# --------------------------------------------------------------------------- +# FR-010 (MC1): missing API key fail-fast +# --------------------------------------------------------------------------- + + +class TestFR010MissingApiKey: + def test_invoke_skill_raises_setup_error_without_key( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """FR-010: invoke_skill MUST raise SetupError when the key is absent.""" + import asyncio + + from tests.darnit.parity.tier2.claude_agent_sdk_client import ( + SetupError, + invoke_skill, + ) + + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + with pytest.raises(SetupError, match="ANTHROPIC_API_KEY"): + asyncio.new_event_loop().run_until_complete( + invoke_skill(fixture_dir=Path.cwd()), + ) + + def test_run_py_exits_3_without_api_key( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """FR-010: run.py subprocess exit code is 3 (SETUP) when key absent. + + The runner also fails setup=3 when no fixtures match, but this test + forces a real skill invocation by NOT using --dry-run and pointing at + the actual corpus so the SDK path is exercised. + """ + env = {k: v for k, v in os.environ.items() if k != "ANTHROPIC_API_KEY"} + env["PYTHONPATH"] = str(Path.cwd()) + + rc = subprocess.run( + [ + sys.executable, + "-m", + "tests.darnit.parity.tier2.run", + "--fixture-glob", + "all_pass_repo", + "--artifact-dir", + str(tmp_path / "artifacts"), + ], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + # 3 = SETUP; the runner should NOT proceed past the credential check. + assert rc.returncode == 3, f"expected exit 3, got {rc.returncode}\nstderr: {rc.stderr}" diff --git a/tests/darnit/parity/tier2/test_skill_markdown_parser.py b/tests/darnit/parity/tier2/test_skill_markdown_parser.py new file mode 100644 index 0000000..c1ad5b0 --- /dev/null +++ b/tests/darnit/parity/tier2/test_skill_markdown_parser.py @@ -0,0 +1,83 @@ +"""Tests for the skill Markdown parser (feature 028 T017). + +Golden-file tests against captured skill outputs plus adversarial cases. +""" + +from __future__ import annotations + +from pathlib import Path + +from tests.darnit.parity.tier2.skill_markdown_parser import SkillReport + +GOLDEN_DIR = Path(__file__).parent / "golden" + + +def _load_golden(name: str) -> str: + return (GOLDEN_DIR / name).read_text() + + +class TestGoldenFiles: + def test_all_pass_parseable(self) -> None: + md = _load_golden("all_pass.md") + r = SkillReport.parse(md) + assert r.parseable + assert r.counts is not None + # The all_pass golden uses verbose "Passed: 2" -- counts key = 'pass' + assert r.counts.get("pass") == 2 + assert r.controls is not None + assert len(r.controls) == 2 + ids = {c.id for c in r.controls} + assert ids == {"OSPS-DO-01.01", "OSPS-LE-03.01"} + + def test_mixed_drift_parseable(self) -> None: + md = _load_golden("mixed_drift.md") + r = SkillReport.parse(md) + assert r.parseable + assert r.counts is not None + assert r.counts.get("pass") == 51 + assert r.counts.get("fail") == 5 + assert r.counts.get("warn") == 7 + assert r.controls is not None + ids = {c.id for c in r.controls} + assert "OSPS-BR-06.01" in ids + assert "OSPS-VM-04.01" in ids + # Extract statuses; PASS controls aren't enumerated in this golden. + for claim in r.controls: + if claim.id in ("OSPS-BR-06.01", "OSPS-VM-04.01"): + assert claim.status == "FAIL" + + def test_unparseable_is_captured_as_such(self) -> None: + md = _load_golden("unparseable.md") + r = SkillReport.parse(md) + assert r.parseable is False + assert r.raw_markdown == md # preserved + assert r.parse_notes # explains why + + +class TestRedactionSanity: + """Regression guard: parsed output MUST NOT contain credential-shaped substrings.""" + + def test_parser_output_does_not_expose_secret_that_isnt_in_input(self) -> None: + """Sanity: if the input has no secret, the parsed SkillReport should + not conjure one. Guards against a future bug where the parser + interpolated env vars into its output.""" + secret = "sk-ant-DISTINCTIVE-TEST-KEY-XYZ" + input_md = "PASS: 5, FAIL: 0\n\n**OSPS-DO-01.01**: PASS" + r = SkillReport.parse(input_md) + # No secret in input, so no secret in output. + assert secret not in str(r) + assert secret not in r.raw_markdown + + +class TestParserRobustness: + def test_never_raises_on_empty_string(self) -> None: + r = SkillReport.parse("") + assert r.parseable is False + + def test_never_raises_on_only_whitespace(self) -> None: + r = SkillReport.parse(" \n\n\t ") + assert r.parseable is False + + def test_never_raises_on_garbage(self) -> None: + r = SkillReport.parse("\x00\x01\x02") + assert r.parseable is False diff --git a/tests/darnit/parity/tier2/test_workflow_config.py b/tests/darnit/parity/tier2/test_workflow_config.py new file mode 100644 index 0000000..21ec5ba --- /dev/null +++ b/tests/darnit/parity/tier2/test_workflow_config.py @@ -0,0 +1,103 @@ +"""Governance-critical workflow-config tests (feature 028 T024). + +Parses `.github/workflows/parity-tier2.yml` and asserts the properties +required by contract tier2-workflow.md (T2-1..T2-6, T2-10, T2-11). Also +enforces SC-005a by scanning all workflow files for stray +ANTHROPIC_API_KEY references. + +LC2 fix: pure-Python file iteration, no `subprocess grep`; portable +across Linux/macOS/Windows CI. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +WORKFLOWS_DIR = Path(".github/workflows") +TIER2_WORKFLOW = WORKFLOWS_DIR / "parity-tier2.yml" + + +def _load_workflow() -> dict: + """Parse the Tier 2 workflow YAML.""" + try: + import yaml + except ImportError: # pragma: no cover + pytest.skip("PyYAML not installed") + if not TIER2_WORKFLOW.exists(): + pytest.fail(f"Tier 2 workflow missing: {TIER2_WORKFLOW}") + with TIER2_WORKFLOW.open() as f: + return yaml.safe_load(f) + + +class TestGovernanceInvariants: + def test_t2_1_only_workflow_dispatch_trigger(self) -> None: + """T2-1: workflow_dispatch is the ONLY trigger.""" + workflow = _load_workflow() + # PyYAML parses `on:` as True (Python bool) in some versions; + # accept either key. + triggers = workflow.get("on") or workflow.get(True) + assert triggers is not None, "workflow must have an `on:` block" + assert isinstance(triggers, dict) + keys = set(triggers.keys()) + assert keys == {"workflow_dispatch"}, f"only `workflow_dispatch` allowed, found: {keys}" + + def test_t2_2_environment_declared(self) -> None: + """T2-2: job MUST declare `environment: parity-tier2`.""" + workflow = _load_workflow() + jobs = workflow.get("jobs", {}) + assert "tier2" in jobs, "job named `tier2` required" + assert jobs["tier2"].get("environment") == "parity-tier2" + + def test_t2_5_permissions_contents_read_only(self) -> None: + """T2-5: `permissions: contents: read` and NO write scope.""" + workflow = _load_workflow() + jobs = workflow.get("jobs", {}) + tier2 = jobs["tier2"] + perms = tier2.get("permissions") + assert perms is not None, "job must declare `permissions:`" + assert perms.get("contents") == "read", f"contents scope must be read, got {perms.get('contents')!r}" + # No other keys with 'write' value. + for k, v in perms.items(): + assert v != "write", f"forbidden write scope: {k}" + + def test_t2_10_no_api_key_input(self) -> None: + """T2-10: workflow MUST NOT accept an api_key input (governance + regression guard).""" + workflow = _load_workflow() + triggers = workflow.get("on") or workflow.get(True) + dispatch = triggers.get("workflow_dispatch", {}) + inputs = dispatch.get("inputs", {}) if isinstance(dispatch, dict) else {} + for name in inputs or {}: + assert "api_key" not in name.lower(), f"forbidden input {name!r} (T2-10)" + + def test_t2_11_artifact_upload_on_any_exit_code(self) -> None: + """T2-11: upload-artifact runs with `if: always()`.""" + workflow = _load_workflow() + steps = workflow["jobs"]["tier2"]["steps"] + upload_steps = [s for s in steps if isinstance(s, dict) and "uses" in s and "upload-artifact" in s["uses"]] + assert len(upload_steps) >= 1 + for step in upload_steps: + # PyYAML parses `if:` as True key too on some versions; check both. + condition = step.get("if") or step.get(True) + assert condition == "always()" or condition is True, ( + f"upload step must be `if: always()`, got {condition!r}" + ) + + +class TestApiKeyExclusivity: + def test_sc_005a_no_stray_anthropic_key_references(self) -> None: + """SC-005a + T2-4: `ANTHROPIC_API_KEY` MUST NOT appear in any + workflow file other than `parity-tier2.yml`. Pure-Python file + iteration; no `grep` subprocess (LC2).""" + if not WORKFLOWS_DIR.exists(): + pytest.skip("no .github/workflows directory") + + offenders: list[str] = [] + for wf in list(WORKFLOWS_DIR.glob("*.yml")) + list(WORKFLOWS_DIR.glob("*.yaml")): + if wf.name == TIER2_WORKFLOW.name: + continue + if "ANTHROPIC_API_KEY" in wf.read_text(): + offenders.append(str(wf)) + assert not offenders, f"ANTHROPIC_API_KEY MUST only appear in parity-tier2.yml. Offenders: {offenders}" diff --git a/uv.lock b/uv.lock index 3f4cc44..ceec51f 100644 --- a/uv.lock +++ b/uv.lock @@ -262,6 +262,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, ] +[[package]] +name = "claude-agent-sdk" +version = "0.2.134" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "mcp" }, + { name = "sniffio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/4e/4aa8048367c89b9cbb64f4fac58f9f59110f77231d13e728e2844898ea32/claude_agent_sdk-0.2.134.tar.gz", hash = "sha256:acac34b8068dfeb8fceeb8e2e3b3684f146da206094e55a5d34115063f2bafba", size = 312219, upload-time = "2026-08-08T03:00:16.502Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/75/dc52d7f3632e9a5377487158785495ba5ce009adc292b0b61ffa9ce01e3a/claude_agent_sdk-0.2.134-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5c44159a4a8640edf2634fa9653c40023ed548e24e3aaac01fc3fad85685ed0e", size = 80906236, upload-time = "2026-08-08T03:00:20.156Z" }, + { url = "https://files.pythonhosted.org/packages/f9/19/289b9cf3f26154084070a40d9ac54fb4f624e4a36afa76529f5bd1d112f6/claude_agent_sdk-0.2.134-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:4d4e583bdd834b05f88678948b263e57aa81f20a0817f3d25adfe9d8edb50966", size = 85996885, upload-time = "2026-08-08T03:00:24.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/b2/cb099e7b82eb43626e9dcfbf24e09cd760d59ff3b9e498642c4dffa4fa6f/claude_agent_sdk-0.2.134-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:f259671f03823e07544b52ed37e29accd295012e7d9304f5b7c1ec6a3508c758", size = 90326570, upload-time = "2026-08-08T03:00:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/88/17/59e7a9cc6ebebc9708733a03ddd2b9c90051c7de359686e1e14b1b81d2b1/claude_agent_sdk-0.2.134-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:0cba177a0f234dcdb8dfcdd46803fbe0aadd42fbc5aebba41f65f9d00d05a420", size = 91474377, upload-time = "2026-08-08T03:00:32.869Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c6/c5dd0c41ad6822218ed5af47961ba9a70c88d82ca0f0ff1d12cb7d37a543/claude_agent_sdk-0.2.134-py3-none-win_amd64.whl", hash = "sha256:e5eb243dc62a6ea624aadb4227eb2d03d40394c9ae8a43cb5a94d8d76cd2def0", size = 90944935, upload-time = "2026-08-08T03:00:37.112Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -559,6 +577,9 @@ dev = [ { name = "ruff" }, { name = "vulture" }, ] +parity-tier2 = [ + { name = "claude-agent-sdk" }, +] [package.dev-dependencies] dev = [ @@ -575,6 +596,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "claude-agent-sdk", marker = "extra == 'parity-tier2'", specifier = ">=0.1.0" }, { name = "darnit-baseline", editable = "packages/darnit-baseline" }, { name = "darnit-core", editable = "packages/darnit" }, { name = "darnit-core", extras = ["attestation"], marker = "extra == 'attestation'", editable = "packages/darnit" }, @@ -586,7 +608,7 @@ requires-dist = [ { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8.0" }, { name = "vulture", marker = "extra == 'dev'", specifier = ">=2.11" }, ] -provides-extras = ["attestation", "dev"] +provides-extras = ["attestation", "dev", "parity-tier2"] [package.metadata.requires-dev] dev = [