From b8b0a0f021efdab0b876b2aa620f3a9025cb020f Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 19:06:47 +0300 Subject: [PATCH 01/11] =?UTF-8?q?feat(triage):=20M1=20contract=20=E2=80=94?= =?UTF-8?q?=20migration,=20enums,=20TriageOutput,=20derivation=20(PRD-0008?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Freeze the V1↔V2 triage contract (ADR-0051 / IMPL-0024 M1): - migration 024: rebuild finding.exception_reason CHECK to add `unexploitable` (table rebuild with FK off so the workspace child rows aren't cascade-deleted); add a nullable JSON `triage` column to sidebar_state. - ExceptionReason += unexploitable; IssueStage += triaging, triage_verdict, unexploitable (backend models + frontend api/client.ts). - TriageOutput schema (+ nested reachability/exploitability/report/checks) with a coherent verdict↔recommended_close invariant; report_triager registered in AGENT_OUTPUT_SCHEMAS. - SidebarState.triage section + repo_sidebar persistence + _map_triage (disjoint from `evidence`, overwrites on re-run). - issue_derivation: triaging / triage_verdict / unexploitable stages; an untriaged `new` finding never derives to a remediation stage (the Plan gate). - Remove the enricher's new→triaged auto-advance (ADR-0040 §9 amendment); triaged now means "verdict confirmed real." Deviation from IMPL-0024 §3.1: the sidebar store is columnar, so the triage section needs a migration (it claimed none) — done in 024. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/agents/executor.py | 7 +- backend/cliff/agents/schemas.py | 131 +++++++++++- backend/cliff/agents/sidebar_mapper.py | 13 ++ .../db/migrations/024_triage_contract.sql | 83 ++++++++ backend/cliff/db/repo_sidebar.py | 8 +- backend/cliff/models/__init__.py | 4 + backend/cliff/models/finding.py | 19 +- backend/cliff/models/issue_derivation.py | 55 +++++ backend/tests/api/openapi_snapshot.json | 39 +++- backend/tests/db/test_repo_sidebar_triage.py | 66 ++++++ backend/tests/test_issue_derivation.py | 110 ++++++++++ backend/tests/test_migration_024.py | 196 ++++++++++++++++++ backend/tests/test_routes_findings_reject.py | 19 ++ backend/tests/test_sidebar_mapper.py | 52 +++++ backend/tests/test_status_advance.py | 13 +- backend/tests/test_triage_schema.py | 129 ++++++++++++ frontend/src/api/client.ts | 86 +++++++- 17 files changed, 1012 insertions(+), 18 deletions(-) create mode 100644 backend/cliff/db/migrations/024_triage_contract.sql create mode 100644 backend/tests/db/test_repo_sidebar_triage.py create mode 100644 backend/tests/test_migration_024.py create mode 100644 backend/tests/test_triage_schema.py diff --git a/backend/cliff/agents/executor.py b/backend/cliff/agents/executor.py index b6022a3a..677cbd04 100644 --- a/backend/cliff/agents/executor.py +++ b/backend/cliff/agents/executor.py @@ -236,8 +236,13 @@ def _load_workspace_data( } # Agents that unconditionally advance status (forward-only check still applies). +# +# ADR-0051 §6 amends ADR-0040 §9 / BACKLOG WP6: the finding enricher NO LONGER +# auto-advances ``new → triaged``. ``triaged`` now means "triage confirmed the +# finding is real," set only on human confirmation of a `real` verdict — +# enrichment is an input to triage, not a triage verdict. Triage is a gate, so +# an issue cannot enter the remediation (Plan) flow without that confirmation. _AGENT_STATUS_ADVANCE: dict[str, str] = { - "finding_enricher": "triaged", "remediation_planner": "in_progress", } diff --git a/backend/cliff/agents/schemas.py b/backend/cliff/agents/schemas.py index 9651bcf9..a0dee63a 100644 --- a/backend/cliff/agents/schemas.py +++ b/backend/cliff/agents/schemas.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator # --------------------------------------------------------------------------- # Common output wrapper (matches ADR-0008 output contract) @@ -129,6 +129,129 @@ class RemediationExecutorOutput(BaseModel): model_config = {"extra": "allow"} +# --------------------------------------------------------------------------- +# Triage (ADR-0051 / PRD-0008) — one schema for both producers +# --------------------------------------------------------------------------- + +#: The four triage verdicts. ``needs_review`` is the non-terminal low-signal +#: gate (no terminal recommendation); the other three are confirmable. +TriageVerdict = Literal["real", "unexploitable", "false_positive", "needs_review"] + +#: The two distinct closes a non-real verdict can recommend. +TriageClose = Literal["false_positive", "unexploitable"] + +#: verdict → the only coherent recommended_close (ADR-0051 §2 pairing table). +_VERDICT_TO_CLOSE: dict[str, str | None] = { + "real": None, + "needs_review": None, + "false_positive": "false_positive", + "unexploitable": "unexploitable", +} + + +class TriageReachabilityNode(BaseModel): + """One node in the reachability call-path the panel renders as a chain.""" + + label: str + detail: str | None = None + kind: str | None = None # e.g. "entrypoint" | "step" | "sink" + + model_config = {"extra": "allow"} + + +class TriageReachability(BaseModel): + """Projected from the exposure analyzer's ``reachable`` / + ``reachability_evidence`` (ADR-0042). ``reached=False`` with an empty + ``path`` is the calm "No path found" state (PRD-0008 Story 2).""" + + reached: bool + path: list[TriageReachabilityNode] = Field(default_factory=list) + summary: str | None = None + + model_config = {"extra": "allow"} + + +class TriageExploitability(BaseModel): + """Whether untrusted input can reach the sink. ``unknown`` routes the + verdict to ``needs_review`` (ADR-0051 §9).""" + + exploitable: Literal["yes", "no", "unknown"] + reason: str | None = None + + model_config = {"extra": "allow"} + + +class TriageClaimVsCode(BaseModel): + """The report side-by-side: the reporter's cited snippet vs the actual + repo code (PRD-0008 Story 5).""" + + file: str | None = None + claimed: str | None = None # the reporter's snippet / claim + actual: str | None = None # the real code at the cited location + assessment: str | None = None # one-line judgment + + model_config = {"extra": "allow"} + + +class TriageReport(BaseModel): + """Report-only evidence block; ``None`` for scanner findings.""" + + claim: str | None = None + claim_vs_code: TriageClaimVsCode | None = None + duplicate: bool | None = None + poc_present: bool | None = None + ai_slop_signals: list[str] = Field(default_factory=list) + drafted_reply: str | None = None + + model_config = {"extra": "allow"} + + +class TriageCheck(BaseModel): + """A proof row the panel renders. ``kind`` drives the icon/tone + (pass / warn / fail / info).""" + + eyebrow: str + result: str + kind: str + detail: str | None = None + + model_config = {"extra": "allow"} + + +class TriageOutput(BaseModel): + """The triage verdict (ADR-0051 §2). Emitted by both the deterministic + scanner ``triage_synthesizer`` and the LLM ``report_triager``; the + ``report`` block is populated only for reports. + + ``recommended_close`` is a coherent projection of ``verdict``: it is + filled from the verdict when omitted and rejected when it contradicts the + verdict, so the pairing is a HARD invariant (ADR-0051 §2 / eval + ``pairing_coherent``).""" + + verdict: TriageVerdict + confidence: float = Field(ge=0.0, le=1.0) + recommended_close: TriageClose | None = None + reachability: TriageReachability | None = None + exploitability: TriageExploitability | None = None + report: TriageReport | None = None + checks: list[TriageCheck] = Field(default_factory=list) + + model_config = {"extra": "allow"} + + @model_validator(mode="after") + def _coherent_recommended_close(self) -> TriageOutput: + canonical = _VERDICT_TO_CLOSE[self.verdict] + if self.recommended_close is None: + # Fill the canonical projection so V2 never has to re-derive it. + self.recommended_close = canonical # type: ignore[assignment] + elif self.recommended_close != canonical: + raise ValueError( + f"recommended_close={self.recommended_close!r} is incoherent with " + f"verdict={self.verdict!r} (expected {canonical!r})" + ) + return self + + # Maps agent_type -> the Pydantic model for its structured_output. AGENT_OUTPUT_SCHEMAS: dict[str, type[BaseModel]] = { "finding_enricher": EnrichmentOutput, @@ -138,4 +261,8 @@ class RemediationExecutorOutput(BaseModel): "remediation_planner": PlanOutput, "remediation_executor": RemediationExecutorOutput, "validation_checker": ValidationOutput, + # ADR-0051 — the report triager emits a TriageOutput. The scanner + # synthesizer is a pure function (no agent run), so it is not registered + # here; it constructs + validates TriageOutput directly. + "report_triager": TriageOutput, } diff --git a/backend/cliff/agents/sidebar_mapper.py b/backend/cliff/agents/sidebar_mapper.py index 03d075ba..8983aa79 100644 --- a/backend/cliff/agents/sidebar_mapper.py +++ b/backend/cliff/agents/sidebar_mapper.py @@ -172,6 +172,17 @@ def _map_evidence_collector(out: dict[str, Any]) -> SidebarStateUpdate: ) +def _map_triage(out: dict[str, Any]) -> SidebarStateUpdate: + """Persist the full TriageOutput into the disjoint ``triage`` section + (ADR-0051 §5). A re-run overwrites it (idempotent re-triage). + + Both producers route here: the LLM ``report_triager`` (a registered agent + run, mapped automatically by the executor) and the scanner + ``triage_synthesizer`` (a pure function whose output the triage run path + persists under this key — it is not itself an agent run).""" + return SidebarStateUpdate(triage=out) + + _AGENT_SIDEBAR_MAP: dict[str, Any] = { "finding_enricher": _map_enricher, "owner_resolver": _map_owner, @@ -180,4 +191,6 @@ def _map_evidence_collector(out: dict[str, Any]) -> SidebarStateUpdate: "remediation_planner": _map_planner, "remediation_executor": _map_executor, "validation_checker": _map_validation, + "report_triager": _map_triage, + "triage_synthesizer": _map_triage, } diff --git a/backend/cliff/db/migrations/024_triage_contract.sql b/backend/cliff/db/migrations/024_triage_contract.sql new file mode 100644 index 00000000..6c568cbc --- /dev/null +++ b/backend/cliff/db/migrations/024_triage_contract.sql @@ -0,0 +1,83 @@ +-- 024_triage_contract.sql +-- ADR-0051 / IMPL-0024 M1 — the triage contract. Forward-only (ADR-0033). +-- +-- (1) Rebuild the finding.exception_reason CHECK to add the new +-- 'unexploitable' reason (PRD-0008: "real advisory, not reachable here"). +-- SQLite cannot ALTER a CHECK constraint, so the canonical fix is a +-- table rebuild. Foreign keys MUST be disabled for the rebuild: dropping +-- the old `finding` table with FK enforcement on would cascade through +-- `workspace.finding_id` and delete child workspace rows. +-- +-- (2) Add a nullable JSON `triage` column to sidebar_state so the +-- SidebarState.triage section (TriageOutput) can persist. The sidebar +-- store is columnar (one JSON column per section, like `pull_request` in +-- migration 007), so the additive section needs a column — IMPL-0024 +-- §3.1's "no migration needed" note is corrected here. + +PRAGMA foreign_keys = OFF; + +BEGIN; + +CREATE TABLE finding_new ( + id TEXT PRIMARY KEY, + source_type TEXT NOT NULL, + source_id TEXT NOT NULL, + type TEXT NOT NULL DEFAULT 'dependency', + grade_impact TEXT NOT NULL DEFAULT 'counts', + category TEXT, + assessment_id TEXT REFERENCES assessment(id) ON DELETE CASCADE, + title TEXT NOT NULL, + description TEXT, + plain_description TEXT, + raw_severity TEXT, + normalized_priority TEXT, + status TEXT NOT NULL DEFAULT 'new', + likely_owner TEXT, + why_this_matters TEXT, + asset_id TEXT, + asset_label TEXT, + raw_payload TEXT, + pr_url TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + exception_reason TEXT + CHECK ( + exception_reason IS NULL + OR exception_reason IN ( + 'false_positive', + 'accepted_risk', + 'wont_fix', + 'deferred', + 'unexploitable' + ) + ), + exception_note TEXT +); + +INSERT INTO finding_new ( + id, source_type, source_id, type, grade_impact, category, assessment_id, + title, description, plain_description, raw_severity, normalized_priority, + status, likely_owner, why_this_matters, asset_id, asset_label, raw_payload, + pr_url, created_at, updated_at, exception_reason, exception_note +) +SELECT + id, source_type, source_id, type, grade_impact, category, assessment_id, + title, description, plain_description, raw_severity, normalized_priority, + status, likely_owner, why_this_matters, asset_id, asset_label, raw_payload, + pr_url, created_at, updated_at, exception_reason, exception_note +FROM finding; + +DROP TABLE finding; +ALTER TABLE finding_new RENAME TO finding; + +-- Recreate the indexes the original `finding` table carried. +CREATE UNIQUE INDEX uq_finding_source ON finding(source_type, source_id); +CREATE INDEX idx_finding_type ON finding(type); +CREATE INDEX idx_finding_status ON finding(status); +CREATE INDEX idx_finding_assessment ON finding(assessment_id, type); + +ALTER TABLE sidebar_state ADD COLUMN triage TEXT; -- JSON + +COMMIT; + +PRAGMA foreign_keys = ON; diff --git a/backend/cliff/db/repo_sidebar.py b/backend/cliff/db/repo_sidebar.py index db8f89cc..aaacd2e0 100644 --- a/backend/cliff/db/repo_sidebar.py +++ b/backend/cliff/db/repo_sidebar.py @@ -21,6 +21,7 @@ "validation", "similar_cases", "pull_request", + "triage", ) @@ -38,6 +39,7 @@ def _row_to_sidebar(row: aiosqlite.Row) -> SidebarState: validation=json.loads(row["validation"]) if row["validation"] else None, similar_cases=json.loads(row["similar_cases"]) if row["similar_cases"] else None, pull_request=json.loads(row["pull_request"]) if row["pull_request"] else None, + triage=json.loads(row["triage"]) if row["triage"] else None, updated_at=row["updated_at"], ) @@ -52,8 +54,8 @@ async def upsert_sidebar( INSERT INTO sidebar_state (workspace_id, summary, evidence, owner, plan, definition_of_done, linked_ticket, validation, similar_cases, - pull_request, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + pull_request, triage, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(workspace_id) DO UPDATE SET summary = excluded.summary, evidence = excluded.evidence, @@ -64,6 +66,7 @@ async def upsert_sidebar( validation = excluded.validation, similar_cases = excluded.similar_cases, pull_request = excluded.pull_request, + triage = excluded.triage, updated_at = excluded.updated_at """, ( @@ -77,6 +80,7 @@ async def upsert_sidebar( values["validation"], values["similar_cases"], values["pull_request"], + values["triage"], now, ), ) diff --git a/backend/cliff/models/__init__.py b/backend/cliff/models/__init__.py index d45bd690..f918d8bb 100644 --- a/backend/cliff/models/__init__.py +++ b/backend/cliff/models/__init__.py @@ -219,6 +219,9 @@ class SidebarStateUpdate(BaseModel): validation: dict[str, Any] | None = None similar_cases: dict[str, Any] | None = None pull_request: dict[str, Any] | None = None + # ADR-0051 §5 — the triage verdict (TriageOutput). Disjoint from + # ``evidence``; persisted to its own sidebar_state column (migration 024). + triage: dict[str, Any] | None = None class SidebarState(BaseModel): @@ -232,6 +235,7 @@ class SidebarState(BaseModel): validation: dict[str, Any] | None = None similar_cases: dict[str, Any] | None = None pull_request: dict[str, Any] | None = None + triage: dict[str, Any] | None = None updated_at: datetime diff --git a/backend/cliff/models/finding.py b/backend/cliff/models/finding.py index de563471..84450219 100644 --- a/backend/cliff/models/finding.py +++ b/backend/cliff/models/finding.py @@ -53,12 +53,17 @@ #: PRD-0006 Phase 2 — reason values accepted by ``POST /findings/{id}/reject``. -#: Mirrors the CHECK constraint in migration 012_phase2_columns.sql. +#: Mirrors the CHECK constraint in migration 012_phase2_columns.sql, extended +#: by 024_triage_contract.sql with ``unexploitable`` (ADR-0051 §7 / PRD-0008): +#: a *real* advisory that isn't reachable/exploitable here — kept distinct from +#: ``false_positive`` ("not a real issue"), because they are different truths +#: with different evidence and the dependency flood is almost always the former. ExceptionReason = Literal[ "false_positive", "accepted_risk", "wont_fix", "deferred", + "unexploitable", ] @@ -73,12 +78,21 @@ # Todo "todo", # In progress (agents working, no human gate) + # ADR-0051 / PRD-0008 — triage reasoning is running on an untriaged + # finding (enricher → exposure → synthesis, or report_triager). Same + # in-flight treatment as ``planning`` (cyan pulse). + "triaging", "planning", "generating", "pushing", "opening_pr", "validating", # Review (human gate) + # ADR-0051 / PRD-0008 — triage produced a verdict awaiting the user's + # confirmation (real → accept, needs_review → decide, unexploitable/ + # false_positive → confirm close). Lands in the existing "Needs you" + # section; the verdict value in ``sidebar.triage`` drives the chip copy. + "triage_verdict", "plan_ready", "pr_ready", "pr_awaiting_val", @@ -94,6 +108,9 @@ # Done "fixed", "false_positive", + # ADR-0051 / PRD-0008 — closed as a real advisory that isn't reachable/ + # exploitable here. Distinct Done chip + icon from ``false_positive``. + "unexploitable", "wont_fix", "accepted", "deferred", diff --git a/backend/cliff/models/issue_derivation.py b/backend/cliff/models/issue_derivation.py index a94659ab..1dbc50bc 100644 --- a/backend/cliff/models/issue_derivation.py +++ b/backend/cliff/models/issue_derivation.py @@ -76,6 +76,33 @@ def _pull_request(sidebar: SidebarState | None) -> dict: return sidebar.pull_request +# ADR-0051 / PRD-0008 — agents that produce a triage verdict. A run of any of +# these while the finding is still ``new`` is a triage run: the remediation +# flow only runs them after Start (status is already ``in_progress`` by then), +# so on a ``new`` finding they unambiguously mean "triage is in flight." +_TRIAGE_AGENT_TYPES: tuple[str, ...] = ( + "finding_enricher", + "exposure_analyzer", + "report_triager", +) + + +def _triage_running(latest_runs_by_type: Mapping[str, AgentRun]) -> bool: + return any(_is_running(latest_runs_by_type.get(t)) for t in _TRIAGE_AGENT_TYPES) + + +def _triage_failed(latest_runs_by_type: Mapping[str, AgentRun]) -> bool: + return any(_is_failed(latest_runs_by_type.get(t)) for t in _TRIAGE_AGENT_TYPES) + + +def _has_triage_verdict(sidebar: SidebarState | None) -> bool: + return ( + sidebar is not None + and isinstance(sidebar.triage, dict) + and bool(sidebar.triage.get("verdict")) + ) + + def derive( finding: Finding, *, @@ -112,6 +139,8 @@ def out(section: str, stage: str) -> IssueDerived: ) stage_by_reason = { "false_positive": "false_positive", + # ADR-0051 §7 — distinct Done chip from false_positive. + "unexploitable": "unexploitable", "wont_fix": "wont_fix", "accepted_risk": "accepted", "deferred": "deferred", @@ -190,4 +219,30 @@ def out(section: str, stage: str) -> IssueDerived: # row visibly leaves the Todo section on click. return out("in_progress", "planning") + # ---------- Triage (untriaged finding, ADR-0051 / PRD-0008) ------------- + # Triage runs on a ``new`` finding WITHOUT advancing its status — status + # only moves to ``triaged`` on human confirmation of a ``real`` verdict + # (ADR-0051 §6, which removed the enricher's new→triaged auto-advance). So + # an untriaged finding routes here, never into the remediation flow above: + # Plan is unreachable without a recorded ``real`` verdict (PRD-0008 Story 4). + if finding.status == "new": + # A failed triage run with no verdict yet → Retry affordance in Review + # (never a silent stick in Todo). + if _triage_failed(latest_runs_by_type) and not _has_triage_verdict(sidebar): + return out("review", "failed") + # Triage reasoning in flight (a re-triage beats an existing verdict, + # mirroring the running-planner-beats-existing-plan rule above). + if _triage_running(latest_runs_by_type): + return out("in_progress", "triaging") + # Verdict produced, awaiting the human gate (real → accept, + # needs_review → decide, unexploitable/false_positive → confirm close). + # Lands in the existing "Needs you" section. + if _has_triage_verdict(sidebar): + return out("review", "triage_verdict") + # Untriaged + idle → Todo with a Run-triage action. + return out("todo", "todo") + + # ``triaged`` (verdict confirmed real, remediation not yet started) and any + # other unhandled status fall through to Todo — the user opens the + # workspace to remediate from here. return out("todo", "todo") diff --git a/backend/tests/api/openapi_snapshot.json b/backend/tests/api/openapi_snapshot.json index 53bcba2d..1545c21b 100644 --- a/backend/tests/api/openapi_snapshot.json +++ b/backend/tests/api/openapi_snapshot.json @@ -1966,7 +1966,8 @@ "false_positive", "accepted_risk", "wont_fix", - "deferred" + "deferred", + "unexploitable" ], "type": "string" }, @@ -2197,7 +2198,8 @@ "false_positive", "accepted_risk", "wont_fix", - "deferred" + "deferred", + "unexploitable" ], "type": "string" }, @@ -2416,7 +2418,8 @@ "false_positive", "accepted_risk", "wont_fix", - "deferred" + "deferred", + "unexploitable" ], "type": "string" }, @@ -3051,11 +3054,13 @@ "stage": { "enum": [ "todo", + "triaging", "planning", "generating", "pushing", "opening_pr", "validating", + "triage_verdict", "plan_ready", "pr_ready", "pr_awaiting_val", @@ -3063,6 +3068,7 @@ "failed", "fixed", "false_positive", + "unexploitable", "wont_fix", "accepted", "deferred" @@ -4356,7 +4362,8 @@ "false_positive", "accepted_risk", "wont_fix", - "deferred" + "deferred", + "unexploitable" ], "title": "Reason", "type": "string" @@ -4680,6 +4687,18 @@ ], "title": "Summary" }, + "triage": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Triage" + }, "updated_at": { "format": "date-time", "title": "Updated At", @@ -4807,6 +4826,18 @@ ], "title": "Summary" }, + "triage": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Triage" + }, "validation": { "anyOf": [ { diff --git a/backend/tests/db/test_repo_sidebar_triage.py b/backend/tests/db/test_repo_sidebar_triage.py new file mode 100644 index 00000000..cc4c4a19 --- /dev/null +++ b/backend/tests/db/test_repo_sidebar_triage.py @@ -0,0 +1,66 @@ +"""The sidebar_state.triage column persists + round-trips (migration 024). + +ADR-0051 §5: ``SidebarState.triage`` is a new top-level section. The store is +columnar (one JSON column per section), so this verifies the column added by +migration 024 actually persists and reads back, and stays disjoint from the +``evidence`` section. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from cliff.db.connection import close_db, init_db +from cliff.db.repo_sidebar import get_sidebar, upsert_sidebar +from cliff.models import SidebarStateUpdate + + +@pytest.fixture +async def db(): + conn = await init_db(":memory:") + # sidebar_state.workspace_id FK → workspace(id); create the parent row. + now = datetime.now(UTC).isoformat() + await conn.execute( + "INSERT INTO workspace (id, created_at, updated_at) VALUES ('ws-1', ?, ?)", + (now, now), + ) + await conn.commit() + yield conn + await close_db() + + +async def test_triage_section_round_trips(db) -> None: + triage = { + "verdict": "unexploitable", + "confidence": 0.9, + "recommended_close": "unexploitable", + "reachability": {"reached": False, "path": [], "summary": "No path found."}, + "checks": [{"eyebrow": "REACHABILITY", "result": "No path", "kind": "pass"}], + } + await upsert_sidebar( + db, + "ws-1", + SidebarStateUpdate(evidence={"reachable": "no"}, triage=triage), + ) + sb = await get_sidebar(db, "ws-1") + assert sb is not None + assert sb.triage == triage + # Disjoint from evidence. + assert sb.evidence == {"reachable": "no"} + + +async def test_triage_section_overwrites_on_rerun(db) -> None: + """A re-run overwrites sidebar.triage (idempotent re-triage).""" + await upsert_sidebar( + db, "ws-1", SidebarStateUpdate(triage={"verdict": "needs_review", "confidence": 0.4}) + ) + await upsert_sidebar( + db, "ws-1", SidebarStateUpdate(triage={"verdict": "real", "confidence": 0.95}) + ) + sb = await get_sidebar(db, "ws-1") + assert sb is not None + assert sb.triage is not None + assert sb.triage["verdict"] == "real" + assert sb.triage["confidence"] == 0.95 diff --git a/backend/tests/test_issue_derivation.py b/backend/tests/test_issue_derivation.py index e2cb1705..0178f83e 100644 --- a/backend/tests/test_issue_derivation.py +++ b/backend/tests/test_issue_derivation.py @@ -58,11 +58,13 @@ def make_sidebar( workspace_id: str = "w-1", plan: dict | None = None, pull_request: dict | None = None, + triage: dict | None = None, ) -> SidebarState: return SidebarState( workspace_id=workspace_id, plan=plan, pull_request=pull_request, + triage=triage, updated_at=NOW, ) @@ -719,3 +721,111 @@ def test_no_executor_run_existing_branches_unchanged_regression() -> None: assert result.section == "review" assert result.stage == "plan_ready" + + +# ---------------------------------------------------------------------------- +# Triage stages (ADR-0051 / PRD-0008) +# ---------------------------------------------------------------------------- + + +def test_triage_running_on_new_finding_is_triaging() -> None: + """A triage producer running on an untriaged finding → In progress / + triaging. Triage keeps the finding `new` (it never advances status), so + the run is unambiguously triage, not remediation prep.""" + for agent in ("finding_enricher", "exposure_analyzer", "report_triager"): + result = derive( + make_finding(status="new"), + workspace=make_workspace(), + sidebar=make_sidebar(), + latest_runs_by_type={agent: make_run(agent, "running")}, + ) + assert (result.section, result.stage) == ("in_progress", "triaging"), agent + + +def test_triage_verdict_awaiting_confirm_lands_in_needs_you() -> None: + """Any produced verdict awaiting the human gate → Review (Needs you) / + triage_verdict. The verdict value drives the chip copy in the UI, but all + four route to the same section/stage.""" + for verdict in ("real", "needs_review", "unexploitable", "false_positive"): + result = derive( + make_finding(status="new"), + workspace=make_workspace(), + sidebar=make_sidebar(triage={"verdict": verdict, "confidence": 0.9}), + latest_runs_by_type={}, + ) + assert (result.section, result.stage) == ("review", "triage_verdict"), verdict + + +def test_triage_rerun_in_flight_beats_existing_verdict() -> None: + """A re-triage in flight shows `triaging` even though a prior verdict + exists (mirrors the running-planner-beats-existing-plan rule).""" + result = derive( + make_finding(status="new"), + workspace=make_workspace(), + sidebar=make_sidebar(triage={"verdict": "needs_review", "confidence": 0.4}), + latest_runs_by_type={ + "exposure_analyzer": make_run("exposure_analyzer", "running") + }, + ) + assert (result.section, result.stage) == ("in_progress", "triaging") + + +def test_failed_triage_agent_on_new_surfaces_retry() -> None: + """A failed triage run with no verdict yet surfaces a Retry affordance in + Review (never a silent stick in Todo).""" + result = derive( + make_finding(status="new"), + workspace=make_workspace(), + sidebar=make_sidebar(), + latest_runs_by_type={ + "exposure_analyzer": make_run("exposure_analyzer", "failed") + }, + ) + assert (result.section, result.stage) == ("review", "failed") + + +def test_untriaged_idle_new_finding_is_todo() -> None: + result = derive( + make_finding(status="new"), + workspace=None, + sidebar=None, + latest_runs_by_type={}, + ) + assert (result.section, result.stage) == ("todo", "todo") + + +def test_unexploitable_close_is_distinct_done_stage() -> None: + """ADR-0051 §7 — unexploitable closes render a Done chip distinct from + false_positive.""" + result = derive( + make_finding(status="exception", exception_reason="unexploitable"), + workspace=make_workspace(), + sidebar=make_sidebar(), + latest_runs_by_type={}, + ) + assert (result.section, result.stage) == ("done", "unexploitable") + + +def test_plan_unreachable_without_real_verdict() -> None: + """PRD-0008 Story 4 / gate — an untriaged (`new`) finding never derives to + a remediation/plan stage, even if a plan-shaped sidebar and a completed + planner run are (defensively) present. The verdict awaiting confirmation + keeps it in triage; only confirming `real` (→ status `triaged`) opens the + remediation flow.""" + remediation_stages = { + "planning", "generating", "pushing", "opening_pr", "validating", + "plan_ready", "pr_ready", "pr_awaiting_val", "awaiting_permission", "fixed", + } + result = derive( + make_finding(status="new"), + workspace=make_workspace(), + sidebar=make_sidebar( + plan={"plan_steps": ["x"]}, + triage={"verdict": "real", "confidence": 0.95}, + ), + latest_runs_by_type={ + "remediation_planner": make_run("remediation_planner", "completed"), + }, + ) + assert result.stage not in remediation_stages + assert (result.section, result.stage) == ("review", "triage_verdict") diff --git a/backend/tests/test_migration_024.py b/backend/tests/test_migration_024.py new file mode 100644 index 00000000..3da55355 --- /dev/null +++ b/backend/tests/test_migration_024.py @@ -0,0 +1,196 @@ +"""Tests for SQL migration 024 (ADR-0051 / IMPL-0024 M1 — triage contract). + +Migration 024 does two additive things, both required by the triage step: + + 1. Rebuilds the ``finding.exception_reason`` CHECK constraint to add the new + ``unexploitable`` reason (SQLite cannot ``ALTER`` a CHECK, so the table is + rebuilt). The rebuild MUST preserve every existing row and must not + cascade-delete child rows (``workspace`` references ``finding``). + 2. Adds a nullable JSON ``triage`` column to ``sidebar_state`` so the + ``SidebarState.triage`` section can persist. + +Note: IMPL-0024 §3.1 claimed the sidebar ``triage`` section needed no +migration ("per-workspace JSON context"); the live ``sidebar_state`` schema is +columnar (each section is its own JSON column, e.g. ``pull_request`` added by +migration 007), so a column add is in fact required. This deviation is called +out in the PR. +""" + +from __future__ import annotations + +import shutil +from datetime import UTC, datetime +from pathlib import Path + +import aiosqlite +import pytest + +from cliff.db.migrations import run_migrations + +MIGRATIONS_DIR = Path(__file__).parent.parent / "cliff" / "db" / "migrations" + +_EXISTING_REASONS = ("false_positive", "accepted_risk", "wont_fix", "deferred") + + +def _copy_migrations_upto(dst: Path, upto: int) -> None: + """Copy ``NNN_*.sql`` files whose numeric prefix is ``<= upto`` into *dst*.""" + dst.mkdir(parents=True, exist_ok=True) + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if int(path.name.split("_", 1)[0]) <= upto: + shutil.copy(path, dst / path.name) + + +async def _insert_finding( + db: aiosqlite.Connection, + *, + finding_id: str, + status: str = "new", + exception_reason: str | None = None, + exception_note: str | None = None, +) -> None: + now = datetime.now(UTC).isoformat() + await db.execute( + """ + INSERT INTO finding + (id, source_type, source_id, title, status, + exception_reason, exception_note, created_at, updated_at) + VALUES (?, 'tenable', ?, 'Test finding', ?, ?, ?, ?, ?) + """, + (finding_id, finding_id, status, exception_reason, exception_note, now, now), + ) + await db.commit() + + +async def _table_columns(db: aiosqlite.Connection, table: str) -> dict[str, tuple[str, int]]: + cursor = await db.execute(f"PRAGMA table_info({table})") + rows = await cursor.fetchall() + # row shape: (cid, name, type, notnull, dflt_value, pk) + return {row[1]: (row[2], row[3]) for row in rows} + + +@pytest.fixture +async def db_at_023(tmp_path): + """A FK-enforcing in-memory DB migrated to 023 (pre-triage), with the + migrations dir staged in *tmp_path* so 024 can be applied incrementally.""" + staged = tmp_path / "migrations" + _copy_migrations_upto(staged, 23) + conn = await aiosqlite.connect(":memory:") + conn.row_factory = aiosqlite.Row + await conn.execute("PRAGMA foreign_keys = ON") + await run_migrations(conn, migrations_dir=staged) + try: + yield conn, staged + finally: + await conn.close() + + +async def _apply_024(conn: aiosqlite.Connection, staged: Path) -> int: + """Stage migration 024 alongside the already-applied ones and run it.""" + src = MIGRATIONS_DIR / "024_triage_contract.sql" + assert src.exists(), f"Expected migration file at {src}" + shutil.copy(src, staged / src.name) + return await run_migrations(conn, migrations_dir=staged) + + +async def test_023_rejects_unexploitable_before_rebuild(db_at_023) -> None: + """Guard: the pre-024 CHECK genuinely rejects ``unexploitable`` — so the + 'now accepted' assertion below is meaningful, not vacuous.""" + conn, _ = db_at_023 + with pytest.raises(aiosqlite.IntegrityError): + await _insert_finding( + conn, finding_id="f-pre", status="exception", + exception_reason="unexploitable", + ) + + +async def test_024_applies_and_is_recorded(db_at_023) -> None: + conn, staged = db_at_023 + applied = await _apply_024(conn, staged) + assert applied == 1 + cursor = await conn.execute("SELECT name FROM _migrations") + names = {row[0] for row in await cursor.fetchall()} + assert "024_triage_contract.sql" in names + + +async def test_024_preserves_existing_exception_rows(db_at_023) -> None: + conn, staged = db_at_023 + for reason in _EXISTING_REASONS: + await _insert_finding( + conn, finding_id=f"f-{reason}", status="exception", + exception_reason=reason, exception_note=f"note-{reason}", + ) + await _insert_finding(conn, finding_id="f-new", status="new") + + await _apply_024(conn, staged) + + for reason in _EXISTING_REASONS: + cursor = await conn.execute( + "SELECT status, exception_reason, exception_note FROM finding WHERE id = ?", + (f"f-{reason}",), + ) + row = await cursor.fetchone() + assert tuple(row) == ("exception", reason, f"note-{reason}") + cursor = await conn.execute( + "SELECT status, exception_reason FROM finding WHERE id = 'f-new'" + ) + assert tuple(await cursor.fetchone()) == ("new", None) + + +async def test_024_does_not_cascade_delete_child_workspaces(db_at_023) -> None: + """The finding-table rebuild must run with foreign keys disabled, or + dropping the old ``finding`` table would cascade through the + ``workspace.finding_id`` reference and wipe child rows.""" + conn, staged = db_at_023 + await _insert_finding(conn, finding_id="f-parent", status="new") + now = datetime.now(UTC).isoformat() + await conn.execute( + "INSERT INTO workspace (id, finding_id, created_at, updated_at)" + " VALUES ('ws-child', 'f-parent', ?, ?)", + (now, now), + ) + await conn.commit() + + await _apply_024(conn, staged) + + cursor = await conn.execute( + "SELECT finding_id FROM workspace WHERE id = 'ws-child'" + ) + row = await cursor.fetchone() + assert row is not None, "child workspace row was cascade-deleted by the rebuild" + assert row["finding_id"] == "f-parent" + # FK graph is intact after the rebuild. + cursor = await conn.execute("PRAGMA foreign_key_check") + assert await cursor.fetchall() == [] + + +async def test_024_accepts_unexploitable_reason(db_at_023) -> None: + conn, staged = db_at_023 + await _apply_024(conn, staged) + await _insert_finding( + conn, finding_id="f-unexp", status="exception", + exception_reason="unexploitable", exception_note="air-gapped", + ) + cursor = await conn.execute( + "SELECT exception_reason, exception_note FROM finding WHERE id = 'f-unexp'" + ) + assert tuple(await cursor.fetchone()) == ("unexploitable", "air-gapped") + + +async def test_024_still_rejects_unknown_reason(db_at_023) -> None: + conn, staged = db_at_023 + await _apply_024(conn, staged) + with pytest.raises(aiosqlite.IntegrityError): + await _insert_finding( + conn, finding_id="f-bad", status="exception", exception_reason="bogus", + ) + + +async def test_024_adds_sidebar_triage_column(db_at_023) -> None: + conn, staged = db_at_023 + assert "triage" not in await _table_columns(conn, "sidebar_state") + await _apply_024(conn, staged) + cols = await _table_columns(conn, "sidebar_state") + assert "triage" in cols, "sidebar_state.triage missing after migration 024" + col_type, notnull = cols["triage"] + assert col_type.upper().startswith("TEXT") + assert notnull == 0, "triage must be nullable" diff --git a/backend/tests/test_routes_findings_reject.py b/backend/tests/test_routes_findings_reject.py index f1d7ac28..dde44d6e 100644 --- a/backend/tests/test_routes_findings_reject.py +++ b/backend/tests/test_routes_findings_reject.py @@ -56,6 +56,25 @@ async def test_reject_without_note(db_client, finding_payload) -> None: assert body["derived"]["stage"] == "wont_fix" +async def test_reject_unexploitable_maps_to_distinct_done_stage( + db_client, finding_payload +) -> None: + """ADR-0051 §7 — ``unexploitable`` ("real advisory, not reachable here") is + a distinct close from ``false_positive`` ("not a real issue"), with its own + Done chip.""" + finding = await _create(db_client, finding_payload) + resp = await db_client.post( + f"/api/findings/{finding['id']}/reject", + json={"reason": "unexploitable", "note": "air-gapped; sink unreachable"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["status"] == "exception" + assert body["exception_reason"] == "unexploitable" + assert body["derived"]["section"] == "done" + assert body["derived"]["stage"] == "unexploitable" + + async def test_reject_missing_reason_returns_422(db_client, finding_payload) -> None: finding = await _create(db_client, finding_payload) resp = await db_client.post( diff --git a/backend/tests/test_sidebar_mapper.py b/backend/tests/test_sidebar_mapper.py index 87fd1244..e11f7dcf 100644 --- a/backend/tests/test_sidebar_mapper.py +++ b/backend/tests/test_sidebar_mapper.py @@ -128,6 +128,33 @@ def test_unknown_agent_returns_empty(self): assert update.evidence is None assert update.owner is None + def test_report_triager_maps_triage_section_only(self): + """ADR-0051 §5 — triage lands in its own section, disjoint from the + ``evidence`` section the enricher/exposure/evidence agents share.""" + out = { + "verdict": "unexploitable", + "confidence": 0.88, + "recommended_close": "unexploitable", + "reachability": {"reached": False, "path": [], "summary": "No path found."}, + "checks": [{"eyebrow": "REACHABILITY", "result": "No path", "kind": "pass"}], + } + update = map_to_sidebar_update("report_triager", out) + assert update.triage is not None + assert update.triage["verdict"] == "unexploitable" + # Disjoint from every other section. + assert update.evidence is None + assert update.summary is None + assert update.validation is None + + def test_triage_synthesizer_key_also_maps_triage(self): + """The scanner path persists synthesis output under the + ``triage_synthesizer`` key (it is not a registered agent run).""" + out = {"verdict": "real", "confidence": 0.9} + update = map_to_sidebar_update("triage_synthesizer", out) + assert update.triage is not None + assert update.triage["verdict"] == "real" + assert update.evidence is None + # --------------------------------------------------------------------------- # map_and_upsert (async with mocked DB) @@ -181,6 +208,31 @@ async def test_merge_preserves_existing_fields(self): assert update.evidence["known_exploits"] is True # from enricher assert update.evidence["recommended_urgency"] == "immediate" # from exposure + @pytest.mark.asyncio + async def test_triage_write_preserves_existing_evidence(self): + """Writing the triage section must not clobber the evidence the + enricher/exposure already wrote (ADR-0051 §5: disjoint sections).""" + existing = SidebarState( + workspace_id="ws-1", + summary={"title": "CVE-2026-1234"}, + evidence={"reachable": "likely", "internet_facing": True}, + updated_at=datetime.now(UTC), + ) + mock_db = AsyncMock() + triage_out = {"verdict": "real", "confidence": 0.9, "recommended_close": None} + + with ( + patch("cliff.db.repo_sidebar.get_sidebar", return_value=existing), + patch("cliff.db.repo_sidebar.upsert_sidebar") as mock_upsert, + ): + await map_and_upsert(mock_db, "ws-1", "report_triager", triage_out) + update: SidebarStateUpdate = mock_upsert.call_args[0][2] + assert update.triage is not None + assert update.triage["verdict"] == "real" + # Evidence + summary preserved untouched. + assert update.evidence == {"reachable": "likely", "internet_facing": True} + assert update.summary == {"title": "CVE-2026-1234"} + @pytest.mark.asyncio async def test_first_agent_creates_sidebar(self): """When no sidebar exists yet, create from scratch.""" diff --git a/backend/tests/test_status_advance.py b/backend/tests/test_status_advance.py index abc399f4..46e5dd5f 100644 --- a/backend/tests/test_status_advance.py +++ b/backend/tests/test_status_advance.py @@ -42,7 +42,11 @@ def _make_workspace(finding_id: str = "f-1") -> Workspace: class TestAdvanceFindingStatus: @pytest.mark.asyncio - async def test_enricher_advances_new_to_triaged(self): + async def test_enricher_does_not_auto_advance_new_to_triaged(self): + """ADR-0051 §6 amends ADR-0040 §9 / BACKLOG WP6: the enricher no longer + auto-advances `new → triaged`. ``triaged`` now means "triage confirmed + the finding is real," set only on human confirmation of a `real` + verdict — enrichment is an input to triage, not a verdict.""" db = AsyncMock() with ( patch("cliff.agents.executor.get_workspace", return_value=_make_workspace()), @@ -52,11 +56,8 @@ async def test_enricher_advances_new_to_triaged(self): result = await _advance_finding_status( db, "ws-1", "finding_enricher", {} ) - assert result == "triaged" - mock_upd.assert_called_once() - call_args = mock_upd.call_args - assert call_args[0][1] == "f-1" # finding_id - assert call_args[0][2].status == "triaged" + assert result is None + mock_upd.assert_not_called() @pytest.mark.asyncio async def test_enricher_does_not_regress_in_progress(self): diff --git a/backend/tests/test_triage_schema.py b/backend/tests/test_triage_schema.py new file mode 100644 index 00000000..2c79dd37 --- /dev/null +++ b/backend/tests/test_triage_schema.py @@ -0,0 +1,129 @@ +"""Tests for the TriageOutput contract (ADR-0051 §2 / IMPL-0024 §3.3). + +TriageOutput is the single schema both triage producers emit — the +deterministic scanner ``triage_synthesizer`` and the LLM ``report_triager`` — +and the V1↔V2 seam the frontend consumes. The verdict↔recommended_close +pairing is a HARD invariant (ADR-0051 §2): it is filled from the verdict when +omitted and rejected when it contradicts the verdict. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from cliff.agents.schemas import AGENT_OUTPUT_SCHEMAS, TriageOutput + + +class TestTriageOutputSchema: + def test_scanner_verdict_round_trips(self) -> None: + out = TriageOutput( + verdict="real", + confidence=0.92, + reachability={ + "reached": True, + "path": [ + {"label": "your upload handler", "kind": "entrypoint"}, + {"label": "vulnerable sink", "kind": "sink"}, + ], + "summary": "Untrusted upload reaches the deserialize call.", + }, + exploitability={"exploitable": "yes", "reason": "attacker-controlled bytes"}, + checks=[ + {"eyebrow": "REACHABILITY", "result": "Reachable", "kind": "fail", + "detail": "2-hop path from the upload handler"}, + ], + ) + dumped = out.model_dump() + assert dumped["verdict"] == "real" + assert dumped["recommended_close"] is None + assert dumped["report"] is None + + again = TriageOutput.model_validate(dumped) + assert again.reachability is not None + assert again.reachability.reached is True + assert again.reachability.path[0].label == "your upload handler" + assert again.exploitability is not None + assert again.exploitability.exploitable == "yes" + assert again.checks[0].eyebrow == "REACHABILITY" + + def test_no_path_found_reachability_round_trips(self) -> None: + out = TriageOutput( + verdict="unexploitable", + confidence=0.86, + reachability={"reached": False, "path": [], "summary": "No path found."}, + exploitability={"exploitable": "no", "reason": "vulnerable function never called"}, + ) + again = TriageOutput.model_validate(out.model_dump()) + assert again.reachability is not None + assert again.reachability.reached is False + assert again.reachability.path == [] + assert again.recommended_close == "unexploitable" + + @pytest.mark.parametrize( + "verdict,expected", + [ + ("real", None), + ("needs_review", None), + ("false_positive", "false_positive"), + ("unexploitable", "unexploitable"), + ], + ) + def test_recommended_close_filled_from_verdict(self, verdict, expected) -> None: + out = TriageOutput(verdict=verdict, confidence=0.5) + assert out.recommended_close == expected + + @pytest.mark.parametrize( + "verdict,bad", + [ + ("real", "unexploitable"), + ("real", "false_positive"), + ("needs_review", "unexploitable"), + ("unexploitable", "false_positive"), + ("false_positive", "unexploitable"), + ], + ) + def test_incoherent_recommended_close_rejected(self, verdict, bad) -> None: + with pytest.raises(ValidationError): + TriageOutput(verdict=verdict, confidence=0.5, recommended_close=bad) + + def test_explicit_coherent_close_accepted(self) -> None: + out = TriageOutput( + verdict="unexploitable", confidence=0.8, recommended_close="unexploitable" + ) + assert out.recommended_close == "unexploitable" + + def test_out_of_vocab_verdict_rejected(self) -> None: + with pytest.raises(ValidationError): + TriageOutput(verdict="probably_fine", confidence=0.5) + + @pytest.mark.parametrize("conf", [-0.1, 1.1]) + def test_confidence_out_of_range_rejected(self, conf) -> None: + with pytest.raises(ValidationError): + TriageOutput(verdict="real", confidence=conf) + + def test_report_block_round_trips(self) -> None: + out = TriageOutput( + verdict="false_positive", + confidence=0.81, + report={ + "claim": "RCE via eval() in utils.py", + "claim_vs_code": { + "file": "utils.py", + "claimed": "eval(user_input)", + "actual": "ast.literal_eval(user_input)", + "assessment": "Cited line uses a safe parser, not eval.", + }, + "duplicate": False, + "poc_present": False, + "ai_slop_signals": ["no concrete PoC", "generic CVE prose"], + "drafted_reply": "Thanks for the report — the cited line uses ast.literal_eval.", + }, + ) + d = out.model_dump() + assert d["report"]["claim_vs_code"]["actual"] == "ast.literal_eval(user_input)" + assert d["report"]["ai_slop_signals"] == ["no concrete PoC", "generic CVE prose"] + assert d["recommended_close"] == "false_positive" + + def test_report_triager_registered_with_triage_schema(self) -> None: + assert AGENT_OUTPUT_SCHEMAS["report_triager"] is TriageOutput diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 27a6564c..c8672169 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -28,7 +28,10 @@ export type ExceptionReason = | 'false_positive' | 'accepted_risk' | 'wont_fix' - | 'deferred'; + | 'deferred' + // ADR-0051 §7 / PRD-0008 — a real advisory that isn't reachable/exploitable + // here. Distinct from false_positive ("not a real issue"). + | 'unexploitable'; // PRD-0006 / IMPL-0006 — server-derived UI section + stage. Computed in // repo_finding from workspace + sidebar + agent-run state. Never persisted. @@ -36,7 +39,15 @@ export type IssueSection = 'review' | 'in_progress' | 'todo' | 'done'; export type IssueStage = | 'todo' + // ADR-0051 / PRD-0008 — triage reasoning is running on an untriaged finding + // (enricher → exposure → synthesis, or report_triager). Same in-flight + // treatment as 'planning' (cyan pulse). + | 'triaging' | 'planning' | 'generating' | 'pushing' | 'opening_pr' | 'validating' + // ADR-0051 / PRD-0008 — triage produced a verdict awaiting the user's + // confirmation. Lands in the existing "Needs you" section; the verdict + // value in sidebar.triage drives the chip copy. + | 'triage_verdict' | 'plan_ready' | 'pr_ready' | 'pr_awaiting_val' // Remediation executor parked on an ask-tier tool request — surfaces in // the Review section's "Needs you" bucket. Backend-side this is driven @@ -53,7 +64,11 @@ export type IssueStage = // pill, top widget, and footer all show a terminal-error treatment // instead of an indefinite "Pushing branch / Thinking…" spinner. | 'executor_failed' - | 'fixed' | 'false_positive' | 'wont_fix' | 'accepted' | 'deferred'; + | 'fixed' | 'false_positive' + // ADR-0051 §7 — closed as a real advisory that isn't reachable/exploitable + // here. Distinct Done chip + icon from 'false_positive'. + | 'unexploitable' + | 'wont_fix' | 'accepted' | 'deferred'; export interface IssueDerived { section: IssueSection; @@ -248,6 +263,71 @@ export interface SuggestedNext { action_type: string | null; } +// --------------------------------------------------------------------------- +// Triage (ADR-0051 / PRD-0008 — the V1↔V2 contract). TriageOutput is the +// single shape both producers emit (scanner synthesizer + report triager); +// the `report` block is populated only for source=report findings. +// --------------------------------------------------------------------------- + +export type TriageVerdict = + | 'real' + | 'unexploitable' + | 'false_positive' + | 'needs_review'; + +export type TriageClose = 'false_positive' | 'unexploitable'; + +export interface TriageReachabilityNode { + label: string; + detail?: string | null; + kind?: string | null; +} + +export interface TriageReachability { + reached: boolean; + path: TriageReachabilityNode[]; + summary?: string | null; +} + +export interface TriageExploitability { + exploitable: 'yes' | 'no' | 'unknown'; + reason?: string | null; +} + +export interface TriageClaimVsCode { + file?: string | null; + claimed?: string | null; + actual?: string | null; + assessment?: string | null; +} + +export interface TriageReport { + claim?: string | null; + claim_vs_code?: TriageClaimVsCode | null; + duplicate?: boolean | null; + poc_present?: boolean | null; + ai_slop_signals: string[]; + drafted_reply?: string | null; +} + +export interface TriageCheck { + eyebrow: string; + result: string; + kind: string; + detail?: string | null; +} + +export interface TriageOutput { + verdict: TriageVerdict; + /** 0.0–1.0; render as word + % (e.g. "High · 92%"), never bare. */ + confidence: number; + recommended_close: TriageClose | null; + reachability?: TriageReachability | null; + exploitability?: TriageExploitability | null; + report?: TriageReport | null; + checks: TriageCheck[]; +} + export interface SidebarState { workspace_id: string; summary: Record | null; @@ -259,6 +339,8 @@ export interface SidebarState { validation: Record | null; similar_cases: Record | null; pull_request: Record | null; + // ADR-0051 §5 — the triage verdict (TriageOutput). Disjoint from `evidence`. + triage: TriageOutput | null; updated_at: string; } From 67c4ec4ae7d18444b7c3ee93e04fa50cecb9f93c Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 19:25:59 +0300 Subject: [PATCH 02/11] =?UTF-8?q?feat(triage):=20M2=20scanner=20triage=20?= =?UTF-8?q?=E2=80=94=20synthesizer,=20run=20path,=20deterministic=20eval?= =?UTF-8?q?=20(PRD-0008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - triage_synthesizer: pure function projecting enricher + exposure output to a TriageOutput verdict (ADR-0051 §3). Reachability-first so a reachable finding is never false-cleared even when the enricher abstained; needs_review when confidence < 0.70 or exploitability unknown (§9). - triage_runner.run_triage: reuses the agent-execution machinery — scanner runs enricher → exposure → synthesis, dual-persisting the verdict (a triage_synthesizer agent_run card + sidebar.triage); report dispatches to report_triager (wired in M3). Never advances Finding.status. - POST /api/findings/{id}/triage: ensures a non-status-advancing workspace (the Plan gate) and runs triage in the background. (Deviation from IMPL-0024 §3.2's /workspaces/{id}/triage path — finding-scoped so it can create a workspace without flipping status; documented on the endpoint.) - context_builder.create_workspace gains advance_status (default True); triage passes False so the finding stays `new` until a real verdict is confirmed. - Deterministic eval (ADR-0050 Lane 1, $0, keyless): triage_synthesizer.jsonl + run_triage_synthesis_eval with the asymmetric false-clear HARD gate; a teeth test proves the gate fails a mislabeled real finding. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agents/runtime/triage_synthesizer.py | 267 ++++++++++++++++++ backend/cliff/agents/triage_runner.py | 189 +++++++++++++ backend/cliff/api/routes/agent_execution.py | 80 +++++- backend/cliff/evals/__init__.py | 7 +- backend/cliff/evals/cases.py | 2 + backend/cliff/evals/runners.py | 75 ++++- backend/cliff/workspace/context_builder.py | 13 +- backend/tests/agents/eval/README.md | 8 + .../agents/eval/triage_synthesizer.jsonl | 16 ++ .../agents/test_evals_triage_synthesizer.py | 42 +++ backend/tests/agents/test_triage_runner.py | 167 +++++++++++ .../tests/agents/test_triage_synthesizer.py | 189 +++++++++++++ backend/tests/api/openapi_snapshot.json | 61 ++++ backend/tests/test_routes_agent_execution.py | 84 +++++- 14 files changed, 1193 insertions(+), 7 deletions(-) create mode 100644 backend/cliff/agents/runtime/triage_synthesizer.py create mode 100644 backend/cliff/agents/triage_runner.py create mode 100644 backend/tests/agents/eval/triage_synthesizer.jsonl create mode 100644 backend/tests/agents/test_evals_triage_synthesizer.py create mode 100644 backend/tests/agents/test_triage_runner.py create mode 100644 backend/tests/agents/test_triage_synthesizer.py diff --git a/backend/cliff/agents/runtime/triage_synthesizer.py b/backend/cliff/agents/runtime/triage_synthesizer.py new file mode 100644 index 00000000..a8c8225e --- /dev/null +++ b/backend/cliff/agents/runtime/triage_synthesizer.py @@ -0,0 +1,267 @@ +"""Scanner triage synthesizer (ADR-0051 §3) — a pure function, no LLM. + +The verdict for a *scanner* finding is a deterministic projection of work the +finding enricher (ADR-0040) and exposure analyzer (ADR-0042) already did. This +keeps scanner triage explainable (the proof IS the exposure evidence), $0, and +testable without a model (ADR-0050 Lane 1). + +Mapping (ADR-0051 §3), evaluated **reachability-first** so a finding the +exposure analyzer reports as reachable is never closed as ``false_positive`` +even when the enricher abstained — false-clearing a real finding is the +load-bearing failure (ADR-0051 §10): + + missing exposure → needs_review (never a confident clear) + not reachable / no path → unexploitable + reachable + internet-facing → real + reachable + internal/unknown facing → needs_review (deployment-dependent) + reachability undetermined, real advisory → needs_review + reachability undetermined, no advisory → false_positive (enricher abstention) + +``needs_review`` is produced whenever confidence would fall below the +``NEEDS_REVIEW_CONFIDENCE`` threshold or exploitability is ``unknown`` +(ADR-0051 §9). The threshold is a tunable constant, not a magic number frozen +in the branches. +""" + +from __future__ import annotations + +from typing import Any + +from cliff.agents.schemas import TriageOutput + +#: Below this confidence (or when exploitability is unknown) the synthesizer +#: routes to ``needs_review`` rather than a terminal verdict (ADR-0051 §9). +#: Tuned against the eval harness — see backend/tests/agents/eval. +NEEDS_REVIEW_CONFIDENCE = 0.70 + +# Confidence anchors per verdict (all confident verdicts sit at/above the +# threshold; every needs_review sits below it — see the §9 invariant test). +_CONF_REAL = 0.82 +_CONF_REAL_KNOWN_EXPLOIT = 0.92 +_CONF_UNEXPLOITABLE_FACING_KNOWN = 0.88 +_CONF_UNEXPLOITABLE_FACING_UNKNOWN = 0.78 +_CONF_FALSE_POSITIVE = 0.74 +_CONF_NEEDS_REVIEW = 0.55 +_CONF_MISSING_EXPOSURE = 0.40 + +# Free-text ``ExposureOutput.reachable`` classification. Order matters: +# negatives, then uncertainty, then affirmatives, then a conservative default. +_REACH_NEGATIVE = ( + "no path", + "not reachable", + "unreachable", + "not exploitable", + "cannot be reached", + "no reachable path", +) +_REACH_UNCERTAIN = ( + "unlikely", + "unknown", + "unclear", + "depends", + "maybe", + "possibl", # possible / possibly + "indeterminate", + "uncertain", +) +_REACH_AFFIRMATIVE = ( + "reachable", + "yes", + "true", + "confirmed", + "likely", + "direct", + "exploitable", +) + + +def _classify_reachable(reachable: str | None) -> str: + """Map the analyzer's free-text ``reachable`` to ``yes`` / ``no`` / + ``unknown``. An empty value means the analyzer ran but couldn't determine + reachability → ``unknown`` (the *missing exposure* case is handled by the + caller before this is reached).""" + if not reachable: + return "unknown" + s = reachable.strip().lower() + if any(k in s for k in _REACH_NEGATIVE) or s in ("no", "false", "none"): + return "no" + if any(k in s for k in _REACH_UNCERTAIN): + return "unknown" + if any(k in s for k in _REACH_AFFIRMATIVE): + return "yes" + return "unknown" + + +def _enricher_abstained(enrichment: dict[str, Any] | None) -> bool: + """True when the enricher found no public advisory substantiating a real + vulnerability — no CVE and no CVSS score (ADR-0051 §3 abstention signal).""" + if not enrichment: + return True + return not (enrichment.get("cve_ids") or []) and enrichment.get("cvss_score") is None + + +def synthesize_triage( + enrichment: dict[str, Any] | None, + exposure: dict[str, Any] | None, +) -> TriageOutput: + """Project a triage verdict from recorded enricher + exposure output.""" + known_exploits = bool((enrichment or {}).get("known_exploits")) + + # Missing exposure entirely → never a confident clear (ADR-0051 §10). + if exposure is None: + return TriageOutput( + verdict="needs_review", + confidence=_CONF_MISSING_EXPOSURE, + exploitability={ + "exploitable": "unknown", + "reason": "Exposure analysis hasn't run yet — reachability is undetermined.", + }, + checks=[ + { + "eyebrow": "REACHABILITY", + "result": "Not yet analyzed", + "kind": "warn", + "detail": "Run the exposure analyzer to determine reachability.", + } + ], + ) + + reachable_raw = exposure.get("reachable") + reached = _classify_reachable(reachable_raw if isinstance(reachable_raw, str) else None) + internet_facing = exposure.get("internet_facing") + evidence = exposure.get("reachability_evidence") + + verdict: str + confidence: float + exploitability: dict[str, Any] + + if reached == "no": + verdict = "unexploitable" + confidence = ( + _CONF_UNEXPLOITABLE_FACING_KNOWN + if internet_facing is not None + else _CONF_UNEXPLOITABLE_FACING_UNKNOWN + ) + exploitability = { + "exploitable": "no", + "reason": evidence or "The vulnerable code path is not reachable here.", + } + elif reached == "yes" and internet_facing is True: + verdict = "real" + confidence = _CONF_REAL_KNOWN_EXPLOIT if known_exploits else _CONF_REAL + exploitability = { + "exploitable": "yes", + "reason": evidence + or "Reachable from an internet-facing entrypoint by untrusted input.", + } + elif reached == "yes": + # Reachable but the facing is internal/unknown — depends on deployment. + verdict = "needs_review" + confidence = _CONF_NEEDS_REVIEW + exploitability = { + "exploitable": "unknown", + "reason": "Reachable, but whether untrusted input reaches it depends on " + "the deployment (not internet-facing / facing unknown).", + } + elif _enricher_abstained(enrichment): + # No advisory + reachability undetermined → not a real issue. + verdict = "false_positive" + confidence = _CONF_FALSE_POSITIVE + exploitability = { + "exploitable": "no", + "reason": "No public advisory substantiates this finding.", + } + else: + # A real advisory exists but reachability can't be determined → abstain. + verdict = "needs_review" + confidence = _CONF_NEEDS_REVIEW + exploitability = { + "exploitable": "unknown", + "reason": "A real advisory, but reachability of the sink is undetermined.", + } + + # ADR-0051 §9 — defensive override: a terminal verdict that somehow falls + # below the threshold (or has unknown exploitability) becomes needs_review. + if verdict in ("real", "unexploitable") and ( + confidence < NEEDS_REVIEW_CONFIDENCE + or exploitability.get("exploitable") == "unknown" + ): + verdict = "needs_review" + + reachability = _build_reachability(reached, evidence) + checks = _build_checks(reached, internet_facing, known_exploits, enrichment, evidence) + + return TriageOutput( + verdict=verdict, # type: ignore[arg-type] + confidence=confidence, + reachability=reachability, + exploitability=exploitability, + checks=checks, + ) + + +def _build_reachability(reached: str, evidence: str | None) -> dict[str, Any] | None: + if reached == "no": + return {"reached": False, "path": [], "summary": evidence or "No path found."} + if reached == "yes": + return {"reached": True, "path": [], "summary": evidence} + return None # undetermined — no reachability block to render + + +def _build_checks( + reached: str, + internet_facing: bool | None, + known_exploits: bool, + enrichment: dict[str, Any] | None, + evidence: str | None, +) -> list[dict[str, Any]]: + checks: list[dict[str, Any]] = [] + if reached == "no": + checks.append({ + "eyebrow": "REACHABILITY", "result": "No path found", "kind": "pass", + "detail": evidence or "The vulnerable function is never called from your code.", + }) + elif reached == "yes": + checks.append({ + "eyebrow": "REACHABILITY", "result": "Reachable", "kind": "fail", + "detail": evidence or "Your code reaches the vulnerable function.", + }) + else: + checks.append({ + "eyebrow": "REACHABILITY", "result": "Undetermined", "kind": "warn", + "detail": evidence or "Reachability could not be determined.", + }) + + if internet_facing is True: + checks.append({ + "eyebrow": "NETWORK EXPOSURE", "result": "Internet-facing", "kind": "fail", + "detail": "Untrusted input can reach this component.", + }) + elif internet_facing is False: + checks.append({ + "eyebrow": "NETWORK EXPOSURE", "result": "Internal only", "kind": "pass", + "detail": "Not exposed to the public internet.", + }) + else: + checks.append({ + "eyebrow": "NETWORK EXPOSURE", "result": "Unknown", "kind": "warn", + "detail": "Network exposure depends on the deployment.", + }) + + if known_exploits: + checks.append({ + "eyebrow": "EXPLOIT MATURITY", "result": "Known exploit", "kind": "fail", + "detail": (enrichment or {}).get("exploit_details") + or "A public exploit exists for this advisory.", + }) + + if _enricher_abstained(enrichment): + checks.append({ + "eyebrow": "ADVISORY", "result": "No matching advisory", "kind": "info", + "detail": "No CVE/CVSS was found to substantiate this finding.", + }) + + return checks + + +__all__ = ["NEEDS_REVIEW_CONFIDENCE", "synthesize_triage"] diff --git a/backend/cliff/agents/triage_runner.py b/backend/cliff/agents/triage_runner.py new file mode 100644 index 00000000..56e24abe --- /dev/null +++ b/backend/cliff/agents/triage_runner.py @@ -0,0 +1,189 @@ +"""Triage run path (ADR-0051 / IMPL-0024 V1-4). + +Reuses the workspace agent-execution machinery rather than standing up a second +pipeline. For a **scanner** finding a triage run is enricher → exposure → +deterministic synthesis; for a **report** finding it is a single +``report_triager`` run (which carries its own dual-persist). Either way the +verdict lands in both the chat timeline and ``sidebar.triage`` (the CLAUDE.md +agent-output rule). + +Triage never advances ``Finding.status``: a finding stays ``new`` through +triage and only becomes ``triaged`` on human confirmation of a ``real`` verdict +(the Plan gate — ADR-0051 §6). +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from cliff.agents.runtime.triage_synthesizer import synthesize_triage +from cliff.agents.schemas import TriageOutput +from cliff.agents.sidebar_mapper import map_and_upsert +from cliff.db.repo_agent_run import ( + create_agent_run, + list_latest_runs_by_workspace_ids, + update_agent_run, +) +from cliff.db.repo_finding import get_finding +from cliff.db.repo_sidebar import get_sidebar +from cliff.models import AgentRunCreate, AgentRunUpdate + +if TYPE_CHECKING: + import aiosqlite + + from cliff.models import Workspace + +logger = logging.getLogger(__name__) + +#: ``source_type`` of an inbound vulnerability report (vs a scanner finding). +#: Set by the normalizer's report branch (ADR-0022/0023 extension, M3). +REPORT_SOURCE_TYPE = "report" + +#: The pure-function "agent" key the synthesis output persists under. Not a +#: real agent run — the chat card + sidebar are written here directly. +SYNTHESIZER_AGENT_TYPE = "triage_synthesizer" + +_TERMINAL_FAILURE_STATUSES = ("failed", "rate_limited") + +_VERDICT_LABEL = { + "real": "Real risk", + "unexploitable": "Not exploitable", + "false_positive": "False positive", + "needs_review": "Needs your review", +} + + +def _confidence_word(confidence: float) -> str: + if confidence >= 0.85: + return "High" + if confidence >= 0.70: + return "Medium" + return "Low" + + +def _verdict_summary(triage: TriageOutput) -> str: + """One-line human-readable verdict for the chat-timeline card.""" + pct = round(triage.confidence * 100) + word = _confidence_word(triage.confidence) + reason = "" + if triage.exploitability and triage.exploitability.reason: + reason = triage.exploitability.reason + elif triage.reachability and triage.reachability.summary: + reason = triage.reachability.summary + label = _VERDICT_LABEL.get(triage.verdict, triage.verdict) + line = f"**Triage verdict: {label}** ({word} · {pct}%)." + return f"{line} {reason}".strip() + + +async def _persist_synthesis( + db: aiosqlite.Connection, workspace_id: str, triage: TriageOutput +) -> None: + """Dual-persist the synthesized verdict: a chat-timeline card (a completed + ``triage_synthesizer`` agent_run) + ``sidebar.triage``.""" + dumped = triage.model_dump() + run = await create_agent_run( + db, + workspace_id, + AgentRunCreate(agent_type=SYNTHESIZER_AGENT_TYPE, status="running"), + ) + await update_agent_run( + db, + run.id, + AgentRunUpdate( + status="completed", + summary_markdown=_verdict_summary(triage), + confidence=triage.confidence, + structured_output=dumped, + ), + ) + await map_and_upsert(db, workspace_id, SYNTHESIZER_AGENT_TYPE, dumped) + + +def _structured_output(runs: dict[str, Any], agent_type: str) -> dict[str, Any] | None: + run = runs.get(agent_type) + return run.structured_output if run is not None else None + + +async def run_triage( + executor: Any, + db: aiosqlite.Connection, + workspace: Workspace, + *, + env_vars: dict[str, str], +) -> TriageOutput | None: + """Run triage for *workspace*'s finding. Returns the verdict, or ``None`` + when a prerequisite run failed (the derivation surfaces a Retry CTA).""" + finding = await get_finding(db, workspace.finding_id) + if finding is None: + raise ValueError(f"workspace {workspace.id} has no finding") + + if finding.source_type == REPORT_SOURCE_TYPE: + return await _run_report_triage(executor, db, workspace, env_vars=env_vars) + return await _run_scanner_triage(executor, db, workspace, env_vars=env_vars) + + +async def _run_scanner_triage( + executor: Any, + db: aiosqlite.Connection, + workspace: Workspace, + *, + env_vars: dict[str, str], +) -> TriageOutput | None: + for agent_type in ("finding_enricher", "exposure_analyzer"): + result = await executor.execute( + workspace.id, + agent_type, + db, + workspace_dir=workspace.workspace_dir, + env_vars=env_vars, + ) + status = getattr(result, "status", None) + if result is None or status in _TERMINAL_FAILURE_STATUSES: + logger.warning( + "Triage aborted: %s ended status=%s for workspace %s", + agent_type, status, workspace.id, + ) + return None + + latest = await list_latest_runs_by_workspace_ids(db, [workspace.id]) + runs = latest.get(workspace.id, {}) + enrichment = _structured_output(runs, "finding_enricher") + exposure = _structured_output(runs, "exposure_analyzer") + + triage = synthesize_triage(enrichment, exposure) + await _persist_synthesis(db, workspace.id, triage) + return triage + + +async def _run_report_triage( + executor: Any, + db: aiosqlite.Connection, + workspace: Workspace, + *, + env_vars: dict[str, str], +) -> TriageOutput | None: + """Run the report triager (read-only repo access). It persists its own + chat card + ``sidebar.triage`` via the executor's standard path; we read + the verdict back to return it.""" + result = await executor.execute( + workspace.id, + "report_triager", + db, + workspace_dir=workspace.workspace_dir, + env_vars=env_vars, + ) + status = getattr(result, "status", None) + if result is None or status in _TERMINAL_FAILURE_STATUSES: + logger.warning( + "Report triage aborted: status=%s for workspace %s", status, workspace.id + ) + return None + + sidebar = await get_sidebar(db, workspace.id) + if sidebar is None or not sidebar.triage: + return None + return TriageOutput.model_validate(sidebar.triage) + + +__all__ = ["REPORT_SOURCE_TYPE", "SYNTHESIZER_AGENT_TYPE", "run_triage"] diff --git a/backend/cliff/api/routes/agent_execution.py b/backend/cliff/api/routes/agent_execution.py index e491aab0..f90f6e60 100644 --- a/backend/cliff/api/routes/agent_execution.py +++ b/backend/cliff/api/routes/agent_execution.py @@ -11,11 +11,13 @@ from cliff.agents.errors import AgentBusyError, AgentProcessError from cliff.agents.pipeline import VALID_AGENT_TYPES, suggest_next -from cliff.api.routes.workspaces import _resolve_repo_env_vars +from cliff.agents.triage_runner import run_triage +from cliff.api.routes.workspaces import _resolve_github_repo_url, _resolve_repo_env_vars from cliff.db.connection import get_db from cliff.db.repo_agent_run import get_agent_run, update_agent_run +from cliff.db.repo_finding import get_finding from cliff.db.repo_sidebar import get_sidebar -from cliff.db.repo_workspace import get_workspace +from cliff.db.repo_workspace import get_workspace, list_workspaces from cliff.integrations.github_app.client import check_repo_push_access from cliff.models import AgentRunUpdate, SidebarState @@ -429,6 +431,80 @@ async def _run_pipeline() -> None: ) +# --------------------------------------------------------------------------- +# Run triage (ADR-0051 / PRD-0008) +# --------------------------------------------------------------------------- + + +class TriageRunResponse(BaseModel): + workspace_id: str + status: str + + +@router.post( + "/findings/{finding_id}/triage", + response_model=TriageRunResponse, + status_code=202, +) +async def run_triage_endpoint( + finding_id: str, + request: Request, + db=Depends(get_db), +): + """Run triage on a finding as a background task (ADR-0051 / PRD-0008). + + Ensures a workspace for the finding WITHOUT advancing its status (triage + keeps the finding ``new`` until a `real` verdict is confirmed — the Plan + gate, ADR-0051 §6), then runs ``enricher → exposure → synthesis`` for a + scanner finding or ``report_triager`` for a report. Poll + ``GET /workspaces/{workspace_id}/sidebar`` for the verdict. + + Note (deviation from IMPL-0024 §3.2's ``POST /workspaces/{id}/triage``): + triage is finding-scoped because it must create a *non-status-advancing* + workspace — a workspace-scoped path would require the workspace to already + exist, and creating it via the standard remediation path flips the finding + to ``in_progress``, defeating the gate. + """ + finding = await get_finding(db, finding_id) + if finding is None: + raise HTTPException(status_code=404, detail="Finding not found") + + executor = request.app.state.agent_executor + context_builder = request.app.state.context_builder + + # Ensure a workspace — reuse the one-per-finding workspace if present, else + # create one without advancing the finding's status. + existing = await list_workspaces(db, finding_id=finding_id, limit=1) + if existing: + workspace = existing[0] + else: + repo_url = await _resolve_github_repo_url(db) + workspace = await context_builder.create_workspace( + db, finding, repo_url=repo_url, advance_status=False + ) + + try: + await executor.check_not_busy(db, workspace.id) + except AgentBusyError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + env_vars = await _resolve_repo_env_vars(request, db, workspace=workspace) + + async def _run_in_background() -> None: + try: + await run_triage(executor, db, workspace, env_vars=env_vars) + except (AgentBusyError, AgentProcessError): + logger.exception("Triage failed for workspace %s", workspace.id) + except Exception: + logger.exception( + "Unexpected error in background triage for workspace %s", workspace.id + ) + + asyncio.create_task(_run_in_background()) + + return TriageRunResponse(workspace_id=workspace.id, status="running") + + # --------------------------------------------------------------------------- # Approve plan — release the run-all gate before remediation_executor runs # --------------------------------------------------------------------------- diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py index 2e08e8d4..cf5ee773 100644 --- a/backend/cliff/evals/__init__.py +++ b/backend/cliff/evals/__init__.py @@ -18,7 +18,11 @@ from cliff.evals.cases import EvalCase, dataset_dir, load_cases from cliff.evals.models import eval_runnable, harvest_env, select_eval_model from cliff.evals.registry import AgentEvalSpec, get_spec -from cliff.evals.runners import EvalRunResult, run_enricher_eval +from cliff.evals.runners import ( + EvalRunResult, + run_enricher_eval, + run_triage_synthesis_eval, +) __all__ = [ "AgentEvalSpec", @@ -31,5 +35,6 @@ "load_cases", "run_agent", "run_enricher_eval", + "run_triage_synthesis_eval", "select_eval_model", ] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index f63e513d..934a5f4b 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -47,6 +47,8 @@ class Expected(BaseModel): cvss_score: float | None = None cvss_min: float | None = None cvss_max: float | None = None + # ADR-0051 — triage golden verdict (triage_synthesizer / report_triager). + verdict: str | None = None def as_dict(self) -> dict[str, Any]: """The declared-only mapping the deterministic evaluators consume.""" diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 4e17233f..a2fe4a15 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -180,4 +180,77 @@ async def run_enricher_eval( return result -__all__ = ["EvalRunResult", "run_enricher_eval"] +_TRIAGE_VOCAB = {"real", "unexploitable", "false_positive", "needs_review"} +_CLEARING_VERDICTS = {"unexploitable", "false_positive"} +_VERDICT_TO_CLOSE = { + "real": None, + "needs_review": None, + "false_positive": "false_positive", + "unexploitable": "unexploitable", +} + + +def run_triage_synthesis_eval( + cases: list[EvalCase], + *, + graded_floor: float = 0.9, +) -> EvalRunResult: + """Score the deterministic scanner ``triage_synthesizer`` over *cases* + (ADR-0051 §Evaluation, ADR-0050 Lane 1 — $0, keyless, every CI push). + + Each case's ``finding`` carries ``{enrichment, exposure}``; the function is + pure, so this needs no model/env. + + Hard gates (zero tolerance): + - verdict in vocabulary, + - verdict↔recommended_close pairing coherent, + - **false-clear**: a ``real`` golden verdict closed as + unexploitable/false_positive (the asymmetric, load-bearing failure), + - abstention: ``abstain`` cases must resolve to ``needs_review``. + Graded: exact verdict match against the golden label. + """ + if not cases: + raise ValueError( + "run_triage_synthesis_eval got 0 cases — an empty dataset must fail, " + "not silently report PASS. Check the dataset path / tier filter." + ) + + from cliff.agents.runtime.triage_synthesizer import synthesize_triage + + result = EvalRunResult( + agent="triage_synthesizer", n_cases=len(cases), graded_floor=graded_floor + ) + matches: list[bool] = [] + + for case in cases: + finding = case.finding or {} + out = synthesize_triage(finding.get("enrichment"), finding.get("exposure")) + + if out.verdict not in _TRIAGE_VOCAB: + result.hard_failures.append(f"{case.id}: out-of-vocab verdict {out.verdict!r}") + if out.recommended_close != _VERDICT_TO_CLOSE.get(out.verdict): + result.hard_failures.append( + f"{case.id}: incoherent pairing verdict={out.verdict!r} " + f"close={out.recommended_close!r}" + ) + + golden = case.expected.verdict + if golden == "real" and out.verdict in _CLEARING_VERDICTS: + result.hard_failures.append( + f"{case.id}: FALSE-CLEAR — a real finding was closed as " + f"{out.verdict!r} (asymmetric zero-tolerance gate)" + ) + if case.abstain and out.verdict != "needs_review": + result.hard_failures.append( + f"{case.id}: abstention — expected needs_review, got {out.verdict!r}" + ) + + if golden is not None: + matches.append(out.verdict == golden) + + result.graded_rates = {"verdict_match": sum(matches) / len(matches) if matches else 1.0} + result.est_cost_usd = 0.0 + return result + + +__all__ = ["EvalRunResult", "run_enricher_eval", "run_triage_synthesis_eval"] diff --git a/backend/cliff/workspace/context_builder.py b/backend/cliff/workspace/context_builder.py index 2444fb81..bdae45ac 100644 --- a/backend/cliff/workspace/context_builder.py +++ b/backend/cliff/workspace/context_builder.py @@ -65,6 +65,7 @@ async def create_workspace( *, initial_focus: str | None = None, repo_url: str | None = None, + advance_status: bool = True, ) -> Workspace: """Create a complete workspace: DB row + directory + finding context. @@ -79,6 +80,12 @@ async def create_workspace( over the live integration value so editing the integration mid-flight doesn't silently rebind the workspace to a different repo. + ``advance_status`` flips ``Finding.status`` new/triaged → in_progress + (the remediation "Start" semantics). Triage passes ``False``: a triage + run needs the workspace directory + finding context, but must NOT + advance the finding — it stays ``new`` until a `real` verdict is + confirmed (ADR-0051 §6, the Plan gate). + Returns the fully populated Workspace model. """ # 1. DB row @@ -93,8 +100,10 @@ async def create_workspace( # 1b. Flip Finding.status new/triaged → in_progress so the Issues # page (PRD-0006) moves the row out of Todo on the user's click, # rather than waiting for the first agent run to update it. - # Idempotent — other statuses are left alone. - await mark_started_on_workspace_create(db, finding.id) + # Idempotent — other statuses are left alone. Skipped for triage runs + # (advance_status=False) so the finding stays untriaged until confirmed. + if advance_status: + await mark_started_on_workspace_create(db, finding.id) # 2. Resolve workspace integrations (if vault is configured) for the # manifest agents read for available-tooling context. diff --git a/backend/tests/agents/eval/README.md b/backend/tests/agents/eval/README.md index c374eaa3..92c73bad 100644 --- a/backend/tests/agents/eval/README.md +++ b/backend/tests/agents/eval/README.md @@ -24,3 +24,11 @@ The harness reads its dataset directory from `CLIFF_EVAL_DATASET_DIR`: So the *scoring logic* is open and the *data* stays private. To add a public sample case, append a line here; real/sensitive cases go in `cliff-os/eval`. + +## Datasets here + +| File | Agent | Lane | Notes | +|------|-------|------|-------| +| `finding_enricher.jsonl` | finding_enricher | live (key-gated) | reference impl (ADR-0050 §7) | +| `triage_synthesizer.jsonl` | scanner triage synthesis | **CI (deterministic, $0)** | pure-function mapping (ADR-0051 §3); asymmetric false-clear gate | +| `report_triager.jsonl` | report triager | live (key-gated) | claim-vs-code + `tool_trace` no-auto-reject (ADR-0051 §4) | diff --git a/backend/tests/agents/eval/triage_synthesizer.jsonl b/backend/tests/agents/eval/triage_synthesizer.jsonl new file mode 100644 index 00000000..937e6bf5 --- /dev/null +++ b/backend/tests/agents/eval/triage_synthesizer.jsonl @@ -0,0 +1,16 @@ +// Scanner triage synthesizer eval (ADR-0051 §Evaluation / ADR-0050 Lane 1). +// Deterministic, $0, keyless — runs every CI push. Each case feeds recorded +// enricher + exposure output into synthesize_triage and checks the verdict. +// The asymmetric gate (HARD, zero-tolerance) fails any case where a `real` +// finding is closed as unexploitable/false_positive. `abstain: true` cases +// MUST resolve to needs_review. +{"id": "air-gapped-critical", "tier": "ci", "edge_case": "real advisory, no reachable path", "finding": {"enrichment": {"normalized_title": "RCE in libfoo", "cve_ids": ["CVE-2026-0001"], "cvss_score": 9.8, "known_exploits": true}, "exposure": {"reachable": "No path found from any entrypoint", "internet_facing": false, "reachability_evidence": "The vulnerable parse() is never imported."}}, "expected": {"verdict": "unexploitable"}} +{"id": "reachable-untrusted-internet", "tier": "ci", "edge_case": "reachable + untrusted input reaches sink", "finding": {"enrichment": {"normalized_title": "Deserialization RCE", "cve_ids": ["CVE-2026-0002"], "cvss_score": 9.1, "known_exploits": false}, "exposure": {"reachable": "Reachable from the public upload handler", "internet_facing": true, "reachability_evidence": "upload() -> parse() -> deserialize()"}}, "expected": {"verdict": "real"}} +{"id": "reachable-known-exploit", "tier": "ci", "edge_case": "reachable internet-facing with a public exploit", "finding": {"enrichment": {"normalized_title": "Auth bypass", "cve_ids": ["CVE-2026-0003"], "cvss_score": 8.8, "known_exploits": true, "exploit_details": "Metasploit module exists"}, "exposure": {"reachable": "reachable from the login route", "internet_facing": true}}, "expected": {"verdict": "real"}} +{"id": "reachable-internal-only", "tier": "ci", "edge_case": "reachable but not internet-facing — depends on deployment", "abstain": true, "finding": {"enrichment": {"normalized_title": "SQLi in admin tool", "cve_ids": ["CVE-2026-0004"], "cvss_score": 7.5, "known_exploits": false}, "exposure": {"reachable": "reachable from the internal admin console", "internet_facing": false}}, "expected": {"verdict": "needs_review"}} +{"id": "unknown-provenance-ssti", "tier": "ci", "edge_case": "reachable, input provenance undeterminable", "abstain": true, "finding": {"enrichment": {"normalized_title": "Jinja SSTI", "cve_ids": ["CVE-2026-0005"], "cvss_score": 8.1, "known_exploits": false}, "exposure": {"reachable": "reachable from the template renderer", "internet_facing": null}}, "expected": {"verdict": "needs_review"}} +{"id": "reachability-undetermined", "tier": "ci", "edge_case": "real advisory but reachability unknown", "abstain": true, "finding": {"enrichment": {"normalized_title": "Path traversal", "cve_ids": ["CVE-2026-0006"], "cvss_score": 7.2, "known_exploits": false}, "exposure": {"reachable": "unknown", "internet_facing": true}}, "expected": {"verdict": "needs_review"}} +{"id": "no-cve-first-party", "tier": "ci", "edge_case": "enricher abstained (no advisory), reachability unknown", "finding": {"enrichment": {"normalized_title": "Suspicious eval-like string", "cve_ids": [], "cvss_score": null, "known_exploits": false}, "exposure": {"reachable": "unclear", "internet_facing": null}}, "expected": {"verdict": "false_positive"}} +{"id": "abstained-but-no-path", "tier": "ci", "edge_case": "no advisory but clear no-path — reachability wins", "finding": {"enrichment": {"normalized_title": "Flagged config value", "cve_ids": [], "cvss_score": null, "known_exploits": false}, "exposure": {"reachable": "not reachable", "internet_facing": false}}, "expected": {"verdict": "unexploitable"}} +{"id": "abstained-but-reachable", "tier": "ci", "edge_case": "no advisory but reachable+facing — never false-cleared", "finding": {"enrichment": {"normalized_title": "First-party finding", "cve_ids": [], "cvss_score": null, "known_exploits": false}, "exposure": {"reachable": "reachable from the public API", "internet_facing": true}}, "expected": {"verdict": "real"}} +{"id": "missing-exposure", "tier": "ci", "edge_case": "exposure analysis absent — never a confident clear", "abstain": true, "finding": {"enrichment": {"normalized_title": "Unanalyzed CVE", "cve_ids": ["CVE-2026-0007"], "cvss_score": 6.5, "known_exploits": false}, "exposure": null}, "expected": {"verdict": "needs_review"}} diff --git a/backend/tests/agents/test_evals_triage_synthesizer.py b/backend/tests/agents/test_evals_triage_synthesizer.py new file mode 100644 index 00000000..57a239f0 --- /dev/null +++ b/backend/tests/agents/test_evals_triage_synthesizer.py @@ -0,0 +1,42 @@ +"""CI lane for the scanner triage synthesizer eval (ADR-0051 / ADR-0050 §5). + +The synthesizer is a pure function, so this lane is fully deterministic, $0, +and keyless — it runs on every push (NOT in ``_LIVE_LLM_FILES``). It proves the +ADR-0051 §3 mapping holds on the golden corpus and that the asymmetric +false-clear gate has teeth. +""" + +from __future__ import annotations + +from cliff.evals import load_cases, run_triage_synthesis_eval +from cliff.evals.cases import EvalCase + + +def test_triage_synthesizer_eval_passes() -> None: + cases = load_cases("triage_synthesizer", tier="ci") + assert cases, "no CI cases in the triage_synthesizer dataset" + + result = run_triage_synthesis_eval(cases, graded_floor=0.9) + assert result.passed, "\n" + result.report() + # The corpus is hand-labeled to match the deterministic mapping exactly. + assert result.graded_rates["verdict_match"] == 1.0, "\n" + result.report() + + +def test_false_clear_gate_has_teeth() -> None: + """A mislabeled case — a clearly-unreachable finding labeled ``real`` — + must trip the zero-tolerance false-clear HARD gate, proving the gate + actually fails the run rather than rubber-stamping.""" + mislabeled = EvalCase.model_validate( + { + "id": "mislabeled-real", + "tier": "ci", + "finding": { + "enrichment": {"cve_ids": ["CVE-2026-9999"], "cvss_score": 9.0}, + "exposure": {"reachable": "No path found", "internet_facing": False}, + }, + "expected": {"verdict": "real"}, + } + ) + result = run_triage_synthesis_eval([mislabeled], graded_floor=0.9) + assert not result.passed + assert any("FALSE-CLEAR" in hf for hf in result.hard_failures), result.report() diff --git a/backend/tests/agents/test_triage_runner.py b/backend/tests/agents/test_triage_runner.py new file mode 100644 index 00000000..1f3e54ea --- /dev/null +++ b/backend/tests/agents/test_triage_runner.py @@ -0,0 +1,167 @@ +"""The triage run path (IMPL-0024 V1-4) — scanner orchestration. + +``run_triage`` reuses the workspace agent-execution machinery: for a scanner +finding it runs enricher → exposure (via the injected executor), then the +deterministic synthesizer, dual-persisting the verdict to the chat timeline (a +``triage_synthesizer`` agent_run card) and ``sidebar.triage``. + +Tested with a stub executor that seeds agent_run rows with canned structured +output, so the path runs keyless (no LLM) — the report-triager branch is +covered in M3. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from cliff.agents.triage_runner import run_triage +from cliff.db.connection import close_db, init_db +from cliff.db.repo_agent_run import ( + create_agent_run, + list_agent_runs, + update_agent_run, +) +from cliff.db.repo_finding import create_finding, get_finding +from cliff.db.repo_sidebar import get_sidebar +from cliff.db.repo_workspace import create_workspace +from cliff.models import AgentRunCreate, AgentRunUpdate, FindingCreate, WorkspaceCreate + + +class _StubExecutor: + """Simulates ``AgentExecutor.execute`` by writing a completed agent_run + row with canned structured output for each agent type.""" + + def __init__(self, outputs: dict[str, dict], *, fail: set[str] | None = None): + self.outputs = outputs + self.fail = fail or set() + self.calls: list[str] = [] + + async def check_not_busy(self, db, workspace_id): # noqa: ANN001 + return None + + async def execute(self, workspace_id, agent_type, db, **_kwargs): # noqa: ANN001 + self.calls.append(agent_type) + status = "failed" if agent_type in self.fail else "completed" + run = await create_agent_run( + db, workspace_id, AgentRunCreate(agent_type=agent_type, status="running") + ) + return await update_agent_run( + db, + run.id, + AgentRunUpdate( + status=status, + structured_output=self.outputs.get(agent_type, {}), + ), + ) + + +async def _make_workspace(db, *, source_type: str = "trivy") -> object: + finding = await create_finding( + db, + FindingCreate( + source_type=source_type, + source_id="vuln-001", + title="CVE-2026-0001 in libfoo", + ), + ) + ws = await create_workspace(db, WorkspaceCreate(finding_id=finding.id)) + # Give it a directory so the function mirrors the real call shape. + now = datetime.now(UTC).isoformat() + await db.execute( + "UPDATE workspace SET workspace_dir = ?, updated_at = ? WHERE id = ?", + (f"/tmp/ws/{ws.id}", now, ws.id), + ) + await db.commit() + return finding, ws + + +@pytest.fixture +async def db(): + conn = await init_db(":memory:") + yield conn + await close_db() + + +_ENRICH_REAL = { + "normalized_title": "RCE in libfoo", + "cve_ids": ["CVE-2026-0001"], + "cvss_score": 9.1, + "known_exploits": False, +} +_EXPOSURE_REAL = { + "reachable": "Reachable from the public upload API", + "internet_facing": True, + "reachability_evidence": "upload() → parse() → deserialize()", +} +_EXPOSURE_NOPATH = { + "reachable": "No path found from any entrypoint", + "internet_facing": False, +} + + +async def test_scanner_triage_writes_sidebar_and_chat_card(db) -> None: + _finding, ws = await _make_workspace(db) + executor = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_REAL} + ) + + triage = await run_triage(executor, db, ws, env_vars={}) + + assert triage is not None + assert triage.verdict == "real" + # Ran the scanner pipeline in order. + assert executor.calls == ["finding_enricher", "exposure_analyzer"] + # Sidebar persisted. + sidebar = await get_sidebar(db, ws.id) + assert sidebar is not None and sidebar.triage is not None + assert sidebar.triage["verdict"] == "real" + # Chat timeline card persisted (dual-persist rule). + runs = await list_agent_runs(db, ws.id) + synth = [r for r in runs if r.agent_type == "triage_synthesizer"] + assert len(synth) == 1 + assert synth[0].status == "completed" + assert synth[0].structured_output["verdict"] == "real" + assert synth[0].summary_markdown # a human-readable verdict line + + +async def test_triage_rerun_overwrites_sidebar_verdict(db) -> None: + _finding, ws = await _make_workspace(db) + first = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_REAL} + ) + await run_triage(first, db, ws, env_vars={}) + second = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_NOPATH} + ) + triage = await run_triage(second, db, ws, env_vars={}) + + assert triage is not None and triage.verdict == "unexploitable" + sidebar = await get_sidebar(db, ws.id) + assert sidebar.triage["verdict"] == "unexploitable" + + +async def test_triage_does_not_advance_finding_status(db) -> None: + """Triage keeps the finding `new` — status only advances on human + confirmation of a `real` verdict (the gate).""" + finding, ws = await _make_workspace(db) + executor = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_REAL} + ) + await run_triage(executor, db, ws, env_vars={}) + refreshed = await get_finding(db, finding.id) + assert refreshed.status == "new" + + +async def test_failed_exposure_aborts_without_clearing(db) -> None: + """A failed exposure run produces no verdict (the derivation surfaces a + Retry affordance) — never a silent clear.""" + _finding, ws = await _make_workspace(db) + executor = _StubExecutor( + {"finding_enricher": _ENRICH_REAL}, fail={"exposure_analyzer"} + ) + triage = await run_triage(executor, db, ws, env_vars={}) + assert triage is None + sidebar = await get_sidebar(db, ws.id) + assert sidebar is None or sidebar.triage is None diff --git a/backend/tests/agents/test_triage_synthesizer.py b/backend/tests/agents/test_triage_synthesizer.py new file mode 100644 index 00000000..3bcf6a6c --- /dev/null +++ b/backend/tests/agents/test_triage_synthesizer.py @@ -0,0 +1,189 @@ +"""The deterministic scanner triage synthesizer (ADR-0051 §3 / IMPL-0024 V1-3). + +``synthesize_triage`` is a pure function over recorded ``EnrichmentOutput`` + +``ExposureOutput`` — no LLM, $0, runs in keyless CI. These tests pin the full +ADR-0051 §3 mapping matrix and the ADR-0051 §9 ``needs_review`` threshold. + +Safety note (the asymmetric gate): the mapping is reachability-first, so a +finding the exposure analyzer reports as *reachable* is never closed as +``false_positive`` even when the enricher abstained — false-clearing a real +finding is the load-bearing failure (ADR-0051 §10). +""" + +from __future__ import annotations + +import pytest + +from cliff.agents.runtime.triage_synthesizer import synthesize_triage +from cliff.agents.schemas import TriageOutput + +# Enrichment fixtures. +_WITH_CVE = { + "normalized_title": "RCE in libfoo", + "cve_ids": ["CVE-2026-0001"], + "cvss_score": 9.1, + "known_exploits": False, +} +_WITH_CVE_KNOWN_EXPLOIT = {**_WITH_CVE, "known_exploits": True} +_ABSTAINED = { # enricher found no public advisory substantiating a real vuln + "normalized_title": "Suspicious string in config", + "cve_ids": [], + "cvss_score": None, + "known_exploits": False, +} + + +# (label, enrichment, exposure, expected_verdict, expected_close) +_MATRIX = [ + ( + "air-gapped, clear no-path → unexploitable", + _WITH_CVE, + {"reachable": "No path found from any entrypoint", "internet_facing": False}, + "unexploitable", + "unexploitable", + ), + ( + "not reachable, facing unknown → unexploitable", + _WITH_CVE, + {"reachable": "not reachable", "internet_facing": None}, + "unexploitable", + "unexploitable", + ), + ( + "reachable + internet-facing → real", + _WITH_CVE, + {"reachable": "Reachable from the public upload API", "internet_facing": True}, + "real", + None, + ), + ( + "reachable but internal only → needs_review (deployment-dependent)", + _WITH_CVE, + {"reachable": "reachable", "internet_facing": False}, + "needs_review", + None, + ), + ( + "reachable, facing unknown → needs_review (unknown-provenance SSTI)", + _WITH_CVE, + {"reachable": "reachable", "internet_facing": None}, + "needs_review", + None, + ), + ( + "reachability undetermined, real advisory → needs_review", + _WITH_CVE, + {"reachable": "unknown", "internet_facing": True}, + "needs_review", + None, + ), + ( + "enricher abstained + reachability unknown → false_positive", + _ABSTAINED, + {"reachable": "unclear", "internet_facing": None}, + "false_positive", + "false_positive", + ), + ( + "enricher abstained + clear no-path → unexploitable (reachability wins)", + _ABSTAINED, + {"reachable": "no path", "internet_facing": False}, + "unexploitable", + "unexploitable", + ), + ( + "enricher abstained but reachable+facing → real (never false-clear)", + _ABSTAINED, + {"reachable": "reachable", "internet_facing": True}, + "real", + None, + ), + ( + "missing exposure → needs_review (never a confident clear)", + _WITH_CVE, + None, + "needs_review", + None, + ), +] + + +@pytest.mark.parametrize( + "label,enrichment,exposure,expected_verdict,expected_close", + _MATRIX, + ids=[c[0] for c in _MATRIX], +) +def test_synthesis_mapping_matrix( + label, enrichment, exposure, expected_verdict, expected_close +) -> None: + out = synthesize_triage(enrichment, exposure) + assert isinstance(out, TriageOutput) + assert out.verdict == expected_verdict, label + assert out.recommended_close == expected_close, label + + +def test_needs_review_iff_low_confidence_or_unknown_exploitability() -> None: + """ADR-0051 §9 invariant: a verdict is ``needs_review`` exactly when + confidence < 0.70 OR exploitability is unknown; every confident verdict + (real/unexploitable/false_positive) is ≥ 0.70 with a definite + exploitability or none.""" + for _label, enrichment, exposure, _v, _c in _MATRIX: + out = synthesize_triage(enrichment, exposure) + exploit_unknown = ( + out.exploitability is not None and out.exploitability.exploitable == "unknown" + ) + low_conf = out.confidence < 0.70 + if out.verdict == "needs_review": + assert low_conf or exploit_unknown, _label + else: + assert not low_conf, f"{_label}: confident verdict must be ≥0.70" + assert not exploit_unknown, f"{_label}: confident verdict can't be unknown" + + +def test_known_exploit_raises_confidence_on_real() -> None: + exposure = {"reachable": "reachable from public API", "internet_facing": True} + base = synthesize_triage(_WITH_CVE, exposure) + boosted = synthesize_triage(_WITH_CVE_KNOWN_EXPLOIT, exposure) + assert base.verdict == boosted.verdict == "real" + assert boosted.confidence > base.confidence + + +def test_internet_facing_known_raises_confidence_on_unexploitable() -> None: + known = synthesize_triage( + _WITH_CVE, {"reachable": "no path found", "internet_facing": False} + ) + unknown = synthesize_triage( + _WITH_CVE, {"reachable": "no path found", "internet_facing": None} + ) + assert known.verdict == unknown.verdict == "unexploitable" + assert known.confidence > unknown.confidence + + +def test_no_path_state_sets_reachability_reached_false_with_empty_path() -> None: + out = synthesize_triage( + _WITH_CVE, {"reachable": "No path found", "internet_facing": False} + ) + assert out.reachability is not None + assert out.reachability.reached is False + assert out.reachability.path == [] + + +def test_reachable_state_sets_reachability_reached_true() -> None: + out = synthesize_triage( + _WITH_CVE, + { + "reachable": "reachable", + "internet_facing": True, + "reachability_evidence": "upload() → parse() → deserialize()", + }, + ) + assert out.reachability is not None + assert out.reachability.reached is True + assert out.reachability.summary == "upload() → parse() → deserialize()" + + +def test_output_always_has_proof_checks() -> None: + for _label, enrichment, exposure, _v, _c in _MATRIX: + out = synthesize_triage(enrichment, exposure) + assert out.checks, f"{_label}: synthesis must emit at least one proof row" + assert all(c.kind in {"pass", "warn", "fail", "info"} for c in out.checks) diff --git a/backend/tests/api/openapi_snapshot.json b/backend/tests/api/openapi_snapshot.json index 1545c21b..7f4d8288 100644 --- a/backend/tests/api/openapi_snapshot.json +++ b/backend/tests/api/openapi_snapshot.json @@ -4975,6 +4975,24 @@ "title": "TimeToCloseSeries", "type": "object" }, + "TriageRunResponse": { + "properties": { + "status": { + "title": "Status", + "type": "string" + }, + "workspace_id": { + "title": "Workspace Id", + "type": "string" + } + }, + "required": [ + "workspace_id", + "status" + ], + "title": "TriageRunResponse", + "type": "object" + }, "ValidationError": { "properties": { "ctx": { @@ -6270,6 +6288,49 @@ ] } }, + "/api/findings/{finding_id}/triage": { + "post": { + "description": "Run triage on a finding as a background task (ADR-0051 / PRD-0008).\n\nEnsures a workspace for the finding WITHOUT advancing its status (triage\nkeeps the finding ``new`` until a `real` verdict is confirmed \u2014 the Plan\ngate, ADR-0051 \u00a76), then runs ``enricher \u2192 exposure \u2192 synthesis`` for a\nscanner finding or ``report_triager`` for a report. Poll\n``GET /workspaces/{workspace_id}/sidebar`` for the verdict.\n\nNote (deviation from IMPL-0024 \u00a73.2's ``POST /workspaces/{id}/triage``):\ntriage is finding-scoped because it must create a *non-status-advancing*\nworkspace \u2014 a workspace-scoped path would require the workspace to already\nexist, and creating it via the standard remediation path flips the finding\nto ``in_progress``, defeating the gate.", + "operationId": "run_triage_endpoint_api_findings__finding_id__triage_post", + "parameters": [ + { + "in": "path", + "name": "finding_id", + "required": true, + "schema": { + "title": "Finding Id", + "type": "string" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriageRunResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Run Triage Endpoint", + "tags": [ + "agent-execution" + ] + } + }, "/api/integrations/ai/autodetect": { "get": { "description": "Scan common locations for existing AI keys. Never returns the key.", diff --git a/backend/tests/test_routes_agent_execution.py b/backend/tests/test_routes_agent_execution.py index 777a5f5f..f575afd0 100644 --- a/backend/tests/test_routes_agent_execution.py +++ b/backend/tests/test_routes_agent_execution.py @@ -20,7 +20,7 @@ from cliff.api.routes.agent_execution import router from cliff.db.connection import get_db from cliff.integrations.github_app.client import RepoPushAccess -from cliff.models import AgentRun, Workspace +from cliff.models import AgentRun, Finding, Workspace # --------------------------------------------------------------------------- # App fixture with mock DB dependency @@ -846,3 +846,85 @@ async def _resume(*a, **k): finally: for p in ps: p.stop() + + +# --------------------------------------------------------------------------- +# Run-triage endpoint (ADR-0051 / PRD-0008) +# --------------------------------------------------------------------------- + + +def _make_finding(finding_id="f-1", status="new", source_type="trivy"): + return Finding( + id=finding_id, + source_type=source_type, + source_id="vuln-001", + title="CVE-2026-0001 in libfoo", + status=status, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ) + + +class TestTriageEndpoint: + @pytest.mark.asyncio + async def test_triage_creates_non_advancing_workspace_and_runs(self, app, client): + """The triage run path creates a workspace WITHOUT advancing the + finding's status (the Plan gate) and schedules run_triage.""" + finding = _make_finding(status="new") + ws = _make_workspace() + app.state.context_builder.create_workspace = AsyncMock(return_value=ws) + app.state.agent_executor.check_not_busy = AsyncMock() + + with ( + patch("cliff.api.routes.agent_execution.get_finding", + AsyncMock(return_value=finding)), + patch("cliff.api.routes.agent_execution.list_workspaces", + AsyncMock(return_value=[])), + patch("cliff.api.routes.agent_execution._resolve_github_repo_url", + AsyncMock(return_value=None)), + patch("cliff.api.routes.agent_execution._resolve_repo_env_vars", + AsyncMock(return_value={})), + patch("cliff.api.routes.agent_execution.run_triage", + AsyncMock()) as mock_run, + ): + resp = await client.post(f"/api/findings/{finding.id}/triage") + assert resp.status_code == 202 + assert resp.json()["workspace_id"] == ws.id + assert resp.json()["status"] == "running" + # The Plan gate: the triage workspace must NOT advance the status. + app.state.context_builder.create_workspace.assert_awaited_once() + assert ( + app.state.context_builder.create_workspace.await_args.kwargs[ + "advance_status" + ] + is False + ) + await asyncio.sleep(0.05) + mock_run.assert_awaited_once() + + @pytest.mark.asyncio + async def test_triage_reuses_existing_workspace(self, app, client): + finding = _make_finding(status="new") + ws = _make_workspace() + with ( + patch("cliff.api.routes.agent_execution.get_finding", + AsyncMock(return_value=finding)), + patch("cliff.api.routes.agent_execution.list_workspaces", + AsyncMock(return_value=[ws])), + patch("cliff.api.routes.agent_execution._resolve_repo_env_vars", + AsyncMock(return_value={})), + patch("cliff.api.routes.agent_execution.run_triage", AsyncMock()), + ): + app.state.agent_executor.check_not_busy = AsyncMock() + resp = await client.post(f"/api/findings/{finding.id}/triage") + assert resp.status_code == 202 + assert resp.json()["workspace_id"] == ws.id + # Did not create a second workspace. + app.state.context_builder.create_workspace.assert_not_called() + + @pytest.mark.asyncio + async def test_triage_unknown_finding_returns_404(self, app, client): + with patch("cliff.api.routes.agent_execution.get_finding", + AsyncMock(return_value=None)): + resp = await client.post("/api/findings/does-not-exist/triage") + assert resp.status_code == 404 From 0c01f796dd14d888eadd0aa7680b0bd19c2b1285 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 19:40:05 +0300 Subject: [PATCH 03/11] =?UTF-8?q?feat(triage):=20M3=20report=20triage=20?= =?UTF-8?q?=E2=80=94=20report=5Ftriager=20agent,=20normalizer=20branch,=20?= =?UTF-8?q?live=20eval=20(PRD-0008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - report_triager: the first read-only-tools triage producer (ADR-0051 §4/§8). A Pydantic AI agent with ONLY the `read` tool — no edit/bash/gh(push)/mcp, so it can never mutate the repo, close, or reply. Bounded request budget so a loopy model can't run away (a breach → failed run → Retry, never a clear). System prompt enforces: recommend + draft a reply (human sends), and needs_review when the cited code can't be located (never a confident false_positive). - Executor: a read-only-tools path runs report_triager and persists via the shared _finalize_run (map_and_upsert → _map_triage → sidebar.triage). - Normalizer: source=report is force-tagged source_type='report' so triage routes it to the report triager (pure _resolve_source_type helper). - Registry: report_triager chip. - Eval: report_triager.jsonl (live, key-gated) + run_report_triager_eval with the tool_trace (read-only), false-clear, and abstention HARD gates; keyless tool-surface + source-tagging tests guard the trust boundary in CI. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/agents/executor.py | 49 +++++++ backend/cliff/agents/registry.py | 3 + .../cliff/agents/runtime/report_triager.py | 129 ++++++++++++++++++ backend/cliff/evals/__init__.py | 2 + backend/cliff/evals/cases.py | 4 + backend/cliff/evals/runners.py | 116 +++++++++++++++- backend/cliff/integrations/normalizer.py | 25 +++- backend/tests/agents/conftest.py | 1 + .../tests/agents/eval/report_triager.jsonl | 8 ++ .../tests/agents/test_evals_report_triager.py | 29 ++++ backend/tests/agents/test_report_triager.py | 53 +++++++ 11 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 backend/cliff/agents/runtime/report_triager.py create mode 100644 backend/tests/agents/eval/report_triager.jsonl create mode 100644 backend/tests/agents/test_evals_report_triager.py create mode 100644 backend/tests/agents/test_report_triager.py diff --git a/backend/cliff/agents/executor.py b/backend/cliff/agents/executor.py index 677cbd04..977a945b 100644 --- a/backend/cliff/agents/executor.py +++ b/backend/cliff/agents/executor.py @@ -52,6 +52,7 @@ from cliff.agents.runtime.remediation_executor import ( build_agent as build_executor_agent, ) +from cliff.agents.runtime.report_triager import run_report_triager from cliff.agents.runtime.tools.mcp import build_mcp_toolsets from cliff.agents.sidebar_mapper import map_and_upsert from cliff.db.repo_agent_run import ( @@ -593,6 +594,24 @@ async def execute( "permission request" ) parse_result = outcome.parse_result + elif agent_type == "report_triager": + # ===== Pydantic AI read-only-tools path (ADR-0051 §8) ===== + # The report triager is the one triage producer with tool + # access — but READ-ONLY (the `read` tool, nothing that can + # mutate the repo / close / reply). No permission gating is + # needed because reads are auto-tier. Persistence is identical + # to the no-tools path (map_and_upsert → _map_triage). + parse_result = await self._run_pa_report_triager( + WorkspaceDeps( + workspace_id=workspace_id, + workspace_dir=workspace_dir, + finding=finding_data, + prior_context=prior_ctx, + env_vars=env_vars or {}, + user_note=None, + ), + effective_timeout, + ) else: # ============ Pydantic AI no-tools path ================ # ADR-0047 / IMPL-0022 PR #1 — six no-tools agents run @@ -1036,6 +1055,36 @@ async def _run_pa_no_tools( error=None, ) + async def _run_pa_report_triager( + self, + deps: WorkspaceDeps, + timeout: float, + ) -> ParseResult: + """Run the read-only report triager (ADR-0051 §4) through Pydantic AI. + + Same shape as the no-tools path — a ParseResult carrying the validated + ``TriageOutput`` — so ``_finalize_run`` persists it identically + (``map_and_upsert`` routes ``report_triager`` to ``_map_triage`` → + ``sidebar.triage``). The only difference is the agent has the ``read`` + tool. Provider 429s reuse the same backoff budget via ``_run_pa_call``. + """ + model = await self._resolve_active_model() + structured_output = await self._run_pa_call( + lambda: run_report_triager(deps, model), + agent_type="report_triager", + timeout=timeout, + ) + verdict = structured_output.get("verdict", "needs_review") + return ParseResult( + success=True, + raw_text="", + structured_output=structured_output, + summary=f"Report triage verdict: {verdict}.", + confidence=structured_output.get("confidence"), + suggested_next_action=None, + error=None, + ) + async def _resolve_active_model(self) -> Model: """Resolve the active provider env + model id and build a PA Model. diff --git a/backend/cliff/agents/registry.py b/backend/cliff/agents/registry.py index c9221bef..5dcdd12b 100644 --- a/backend/cliff/agents/registry.py +++ b/backend/cliff/agents/registry.py @@ -24,6 +24,9 @@ class AgentChip: AGENT_CHIPS: list[AgentChip] = [ AgentChip("finding_enricher", "Enrich finding", "search", "enrichment"), AgentChip("exposure_analyzer", "Check exposure", "shield", "exposure"), + # ADR-0051 / PRD-0008 — the report triager (read-only repo) writes the + # triage section for inbound vulnerability reports. + AgentChip("report_triager", "Triage report", "fact_check", "triage"), AgentChip("evidence_collector", "Collect evidence", "biotech", "evidence"), AgentChip("remediation_planner", "Build remediation plan", "checklist", "plan"), AgentChip("remediation_executor", "Remediate", "build", "remediation"), diff --git a/backend/cliff/agents/runtime/report_triager.py b/backend/cliff/agents/runtime/report_triager.py new file mode 100644 index 00000000..6b1dc4fb --- /dev/null +++ b/backend/cliff/agents/runtime/report_triager.py @@ -0,0 +1,129 @@ +"""Report Triager — Pydantic AI runtime (ADR-0051 §4). + +The triage producer for inbound vulnerability *reports* (``source = report``). +Unlike the deterministic scanner synthesizer, free-text claim-vs-code reasoning +can't be projected from structured fields, so this is a real LLM agent — and +the FIRST triage producer with tool access. + +Trust boundary (ADR-0051 §8): **read-only**. It gets the ``read`` tool to +compare the reporter's claim against the cited code and NOTHING ELSE — no +``edit`` / ``bash`` / ``gh`` (push) / ``mcp``. It can therefore never mutate +the repo, close the report, or send a reply. The terminal close/reply is always +a separate, human-confirmed action (PRD-0008 Story 5 — the liability guardrail +that makes serving the report ICP safe). The ``tool_trace`` eval + the +``REPORT_TRIAGER_TOOLS`` test enforce this. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic_ai import Agent +from pydantic_ai.usage import UsageLimits + +from cliff.agents.runtime._prompts import build_user_prompt +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools.read import read +from cliff.agents.schemas import TriageOutput + +if TYPE_CHECKING: + from typing import Any + + from pydantic_ai.models import Model + + +SYSTEM_PROMPT = """\ +You are a vulnerability-report triager for an open-source maintainer. A \ +researcher or bot has submitted a report claiming a vulnerability. Your job is \ +to read the report, check its claim against the maintainer's ACTUAL code, and \ +recommend a verdict — you never decide alone. + +## Your task + +The report (its claim, any cited file/line, and any proof-of-concept) is in the \ +user message. Use the `read` tool to open the cited file(s) and compare what \ +the report claims against what the code actually does. + +Produce a triage verdict with the `report` block populated: +- **claim** — restate the reporter's claim in one plain sentence. +- **claim_vs_code** — the side-by-side: the reporter's cited snippet vs the \ +real code at that location, plus a one-line assessment. +- **duplicate** — is this a known/prior report? +- **poc_present** — did the report include a concrete, runnable proof of \ +concept (not just prose)? +- **ai_slop_signals** — concrete tells of low-quality / AI-generated noise \ +(no PoC, generic CVE boilerplate, cites code that doesn't exist, mismatched \ +language/framework, contradictory claims). Empty if none. +- **drafted_reply** — a courteous reply the maintainer can EDIT and send. \ +Never address it as if already sent. + +Then set: +- **verdict** — `real`, `unexploitable`, `false_positive`, or `needs_review`. +- **confidence** — 0.0–1.0. +- **exploitability** — yes / no / unknown with a one-line reason. + +## Hard rules + +- **You never reject, close, or reply on your own.** You only RECOMMEND a \ +verdict and DRAFT a reply. A human reviews and sends it. Write the reply as a \ +draft, never as a sent message. +- **Read sparingly — at most a handful of `read` calls.** Open the cited \ +file(s) once. If the first read returns "[file not found]", do NOT keep \ +guessing alternative paths: you cannot locate the cited code, so return \ +`needs_review` immediately. +- **When you cannot locate the cited code, do NOT clear the report.** A read \ +miss means you lack the evidence to dismiss it — return `needs_review` (never \ +a confident `false_positive`). Falsely dismissing a real report is the worst \ +outcome. +- **Cite real lines.** Every `claim_vs_code` must reference code you actually \ +read this session. Do not invent file contents. +- **Read-only.** You can only read files. You cannot modify the repo, run \ +commands, or open pull requests — and you must not claim to have done so. +""" + +#: The report triager's COMPLETE tool surface — read-only, by design. Keeping +#: this an explicit constant (vs an inline list in build_agent) gives the +#: trust-boundary test (and reviewers) one obvious thing to assert on. +REPORT_TRIAGER_TOOLS = (read,) + +#: Hard cap on model requests per triage run. The read-only agent only needs a +#: few reads to do the claim-vs-code check; without a cap a weaker model can +#: loop on the `read` tool (guessing paths) and burn requests. A breach +#: surfaces as a failed run (never a confident verdict), which the derivation +#: turns into a Retry — never a silent clear. +REPORT_TRIAGER_REQUEST_LIMIT = 10 + + +def build_agent(model: Model) -> Agent[WorkspaceDeps, TriageOutput]: + """Construct the report triager agent for *model* (read-only repo access).""" + return Agent( + model=model, + output_type=TriageOutput, + deps_type=WorkspaceDeps, + system_prompt=SYSTEM_PROMPT, + tools=list(REPORT_TRIAGER_TOOLS), + ) + + +async def run_report_triager(deps: WorkspaceDeps, model: Model) -> dict[Any, Any]: + """Run the report triager and return its validated TriageOutput as a dict. + + Mirrors :func:`cliff.agents.runtime.no_tools.run_no_tools_agent` so the + executor's persistence path is unchanged — the only difference is the + read tool. Raises whatever Pydantic AI raises; the executor translates the + exception taxonomy.""" + agent = build_agent(model) + result = await agent.run( + build_user_prompt(deps), + deps=deps, + usage_limits=UsageLimits(request_limit=REPORT_TRIAGER_REQUEST_LIMIT), + ) + return result.output.model_dump() + + +__all__ = [ + "REPORT_TRIAGER_TOOLS", + "SYSTEM_PROMPT", + "build_agent", + "run_report_triager", +] diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py index cf5ee773..e765a060 100644 --- a/backend/cliff/evals/__init__.py +++ b/backend/cliff/evals/__init__.py @@ -21,6 +21,7 @@ from cliff.evals.runners import ( EvalRunResult, run_enricher_eval, + run_report_triager_eval, run_triage_synthesis_eval, ) @@ -35,6 +36,7 @@ "load_cases", "run_agent", "run_enricher_eval", + "run_report_triager_eval", "run_triage_synthesis_eval", "select_eval_model", ] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index 934a5f4b..27c50649 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -66,6 +66,10 @@ class EvalCase(BaseModel): abstain: bool = False finding: dict[str, Any] expected: Expected = Field(default_factory=Expected) + # ADR-0051 — report-triager cases stage cited repo files (relpath → text) + # into a temp workspace so the read-only agent can do the claim-vs-code + # check. Omitted (None) for every other agent. + files: dict[str, str] | None = None def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index a2fe4a15..5126ada3 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -253,4 +253,118 @@ def run_triage_synthesis_eval( return result -__all__ = ["EvalRunResult", "run_enricher_eval", "run_triage_synthesis_eval"] +async def run_report_triager_eval( + cases: list[EvalCase], + *, + env: dict[str, str], + model_id: str | None, + graded_floor: float = 0.6, +) -> EvalRunResult: + """Run the REAL report triager over *cases* and score it (ADR-0051 §4 / + ADR-0050 Lane 2 — live, key-gated). + + Each case stages its cited files (``case.files``) into a temp workspace so + the read-only agent can do the claim-vs-code check. + + Hard gates (zero tolerance): + - ``tool_trace``: the report triager exposes NO mutating/side-effecting + tool (it can never auto-close or auto-reply) — ADR-0051 §8, + - **false-clear**: a ``real`` report dismissed as + unexploitable/false_positive (the asymmetric, load-bearing failure), + - abstention: ``abstain`` cases (e.g. a report citing nonexistent code) + must resolve to ``needs_review``, never a confident dismissal. + Graded: exact verdict match against the golden label. + """ + if not cases: + raise ValueError( + "run_report_triager_eval got 0 cases — an empty dataset must fail, " + "not silently report PASS. Check the dataset path / tier filter." + ) + + import tempfile + from pathlib import Path + + from pydantic_ai.exceptions import UsageLimitExceeded + from pydantic_ai.usage import UsageLimits + + from cliff.agents.runtime._prompts import build_user_prompt + from cliff.agents.runtime.deps import WorkspaceDeps + from cliff.agents.runtime.provider import build_model + from cliff.agents.runtime.report_triager import ( + REPORT_TRIAGER_REQUEST_LIMIT, + REPORT_TRIAGER_TOOLS, + build_agent, + ) + from cliff.agents.runtime.tools import bash, edit, gh, webfetch + from cliff.agents.schemas import TriageOutput + + result = EvalRunResult( + agent="report_triager", n_cases=len(cases), graded_floor=graded_floor + ) + + # tool_trace HARD gate (static, but the whole safety story): the report + # triager must carry no tool that can mutate the repo, push, or reply. + mutating = [t for t in (bash, edit, gh, webfetch) if t in REPORT_TRIAGER_TOOLS] + if mutating: + names = ", ".join(t.__name__ for t in mutating) + result.hard_failures.append(f"tool_trace: report_triager exposes {names}") + + model = build_model(env, model_id) + matches: list[bool] = [] + + for case in cases: + with tempfile.TemporaryDirectory() as tmp: + for rel, text in (case.files or {}).items(): + target = Path(tmp) / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + deps = WorkspaceDeps( + workspace_id=case.id, + workspace_dir=tmp, + finding=case.finding, + prior_context={}, + env_vars=env, + user_note=None, + ) + try: + run = await build_agent(model).run( + build_user_prompt(deps), + deps=deps, + usage_limits=UsageLimits( + request_limit=REPORT_TRIAGER_REQUEST_LIMIT + ), + ) + out = run.output + usage = run.usage + result.total_tokens += getattr(usage, "total_tokens", 0) or 0 + except UsageLimitExceeded: + # Ran out of read budget without a conclusion. Production + # surfaces this as a failed run (a Retry), never a clear; the + # eval scores it as the safe abstention so a loopy model can't + # false-clear — but it still counts against verdict_match for + # any non-abstain case. + out = TriageOutput(verdict="needs_review", confidence=0.0) + + golden = case.expected.verdict + if golden == "real" and out.verdict in _CLEARING_VERDICTS: + result.hard_failures.append( + f"{case.id}: FALSE-CLEAR — a real report was dismissed as " + f"{out.verdict!r} (asymmetric zero-tolerance gate)" + ) + if case.abstain and out.verdict != "needs_review": + result.hard_failures.append( + f"{case.id}: abstention — expected needs_review, got {out.verdict!r}" + ) + if golden is not None: + matches.append(out.verdict == golden) + + result.graded_rates = {"verdict_match": sum(matches) / len(matches) if matches else 1.0} + return result + + +__all__ = [ + "EvalRunResult", + "run_enricher_eval", + "run_report_triager_eval", + "run_triage_synthesis_eval", +] diff --git a/backend/cliff/integrations/normalizer.py b/backend/cliff/integrations/normalizer.py index 5db7e0cd..1edd0605 100644 --- a/backend/cliff/integrations/normalizer.py +++ b/backend/cliff/integrations/normalizer.py @@ -238,6 +238,25 @@ """ +#: ADR-0051 / PRD-0008 — an inbound vulnerability report imports under this +#: source. The triage dispatch routes ``source_type == REPORT_SOURCE_TYPE`` to +#: the report_triager (read-only repo) instead of the scanner synthesizer. +REPORT_SOURCE_TYPE = "report" + + +def _resolve_source_type(source: str, item_source_type: str | None) -> str: + """The ``source_type`` a normalized item gets. + + A report import is ALWAYS tagged ``report`` (regardless of what the model + guessed for an unstructured prose item) so triage routes it to the report + triager. Scanner items keep their own ``source_type``, defaulting to the + scanner name when the model left it null. + """ + if source == REPORT_SOURCE_TYPE: + return REPORT_SOURCE_TYPE + return item_source_type or source + + def _build_user_message(source: str, raw_json: str) -> str: """The per-call user message — just the scanner source + raw data. @@ -328,9 +347,9 @@ async def normalize_findings( for i, nf in enumerate(outputs): item = nf.model_dump() # The model_dump always carries every key; fill the load-bearing - # defaults the LLM may have left null. - if item.get("source_type") is None: - item["source_type"] = source + # defaults the LLM may have left null. A report import is force-tagged + # ``source_type='report'`` so triage routes it correctly (ADR-0051). + item["source_type"] = _resolve_source_type(source, item.get("source_type")) # Normalized findings are always brand-new — force the status rather # than trusting the model's value, so a stray string (e.g. "open") # doesn't drop an otherwise-valid finding into ``errors``. diff --git a/backend/tests/agents/conftest.py b/backend/tests/agents/conftest.py index 8eb4bfa5..5ebfbbf8 100644 --- a/backend/tests/agents/conftest.py +++ b/backend/tests/agents/conftest.py @@ -27,6 +27,7 @@ "test_normalizer_agent", "test_plain_description_eval", "test_evals_finding_enricher", + "test_evals_report_triager", ) diff --git a/backend/tests/agents/eval/report_triager.jsonl b/backend/tests/agents/eval/report_triager.jsonl new file mode 100644 index 00000000..9a3332a9 --- /dev/null +++ b/backend/tests/agents/eval/report_triager.jsonl @@ -0,0 +1,8 @@ +// Report triager eval (ADR-0051 §4/§Evaluation / ADR-0050 Lane 2). +// LIVE, key-gated (this agent reasons over free-text + reads cited code). +// `files` stage the cited repo files into a temp workspace. Hard gates: +// tool_trace (read-only), false-clear of a `real` report, abstention to +// needs_review when the cited code can't be located. +{"id": "real-sqli-with-poc", "tier": "live", "edge_case": "reachable SQLi confirmed by reading the cited code", "finding": {"source_type": "report", "source_id": "R-1", "title": "SQL injection in get_user", "description": "get_user() in db.py builds its query with an f-string: f\"SELECT * FROM users WHERE name = '{name}'\". Proof of concept: passing name = ' OR '1'='1 returns every row. This is a classic SQL injection."}, "files": {"db.py": "def get_user(conn, name):\n # build the lookup query\n return conn.execute(f\"SELECT * FROM users WHERE name = '{name}'\").fetchone()\n"}, "expected": {"verdict": "real"}} +{"id": "slop-safe-parser", "tier": "live", "edge_case": "claim contradicted by the cited code (safe parser, no PoC)", "finding": {"source_type": "report", "source_id": "R-2", "title": "Remote code execution via eval() in config loader", "description": "The load_config function in utils.py uses eval() on attacker-controlled input, leading to remote code execution. This is a critical CVE-class vulnerability affecting all users. No proof of concept is included."}, "files": {"utils.py": "import ast\n\n\ndef load_config(raw):\n # parse the config value safely (never eval)\n return ast.literal_eval(raw)\n"}, "expected": {"verdict": "false_positive"}} +{"id": "cites-nonexistent-code", "tier": "live", "edge_case": "report cites a file that does not exist — must abstain", "abstain": true, "finding": {"source_type": "report", "source_id": "R-3", "title": "Path traversal in request handler", "description": "handler.py around line 88 joins a user-supplied path with os.path.join(base_dir, user_path) and serves the result, allowing path traversal (e.g. ../../etc/passwd)."}, "expected": {"verdict": "needs_review"}} diff --git a/backend/tests/agents/test_evals_report_triager.py b/backend/tests/agents/test_evals_report_triager.py new file mode 100644 index 00000000..8c979300 --- /dev/null +++ b/backend/tests/agents/test_evals_report_triager.py @@ -0,0 +1,29 @@ +"""Live lane for the report_triager eval (ADR-0050 §5 Lane 2 / ADR-0051 §4). + +Runs the REAL report triager over the report dataset and scores it. Skipped +automatically when no API key is set (this file is in ``_LIVE_LLM_FILES`` — +see ``conftest.py``); never masks a deterministic lane. + +Hard gates (zero tolerance): ``tool_trace`` (read-only tool surface) + +false-clear of a ``real`` report + abstention on the cites-nonexistent-code +case. Graded: exact verdict match must clear a dataset-wide floor. + +Budget: ~3 LLM calls with a couple of small file reads each — well under the +ADR-0051 §Evaluation per-case ceiling. +""" + +from __future__ import annotations + +from cliff.evals import load_cases, run_report_triager_eval +from tests.agents.eval_utils import LLM_ENV as _LLM_ENV +from tests.agents.eval_utils import LLM_MODEL as _LLM_MODEL + + +async def test_report_triager_eval(): + cases = load_cases("report_triager", tier="live") + assert cases, "no live cases in the active report_triager dataset" + + result = await run_report_triager_eval( + cases, env=_LLM_ENV, model_id=_LLM_MODEL, graded_floor=0.6 + ) + assert result.passed, "\n" + result.report() diff --git a/backend/tests/agents/test_report_triager.py b/backend/tests/agents/test_report_triager.py new file mode 100644 index 00000000..4304bf0e --- /dev/null +++ b/backend/tests/agents/test_report_triager.py @@ -0,0 +1,53 @@ +"""Report triager trust boundary + report-source tagging (ADR-0051 §4/§8). + +Keyless — these assert the *shape* of the report triager (its tool surface and +output type) and the normalizer's report tagging, none of which need an LLM. +The live behaviour (verdict quality, claim-vs-code grounding) is the key-gated +eval in ``test_evals_report_triager.py``. +""" + +from __future__ import annotations + +from cliff.agents.runtime.report_triager import ( + REPORT_TRIAGER_TOOLS, + build_agent, +) +from cliff.agents.runtime.tools import bash, edit, gh, read, webfetch +from cliff.agents.schemas import AGENT_OUTPUT_SCHEMAS, TriageOutput +from cliff.integrations.normalizer import REPORT_SOURCE_TYPE, _resolve_source_type + + +def test_report_triager_tool_surface_is_read_only() -> None: + """The report triager's COMPLETE tool surface is the read tool — it can + never mutate the repo, close the report, push, or reach the network + (ADR-0051 §8 trust boundary / the `tool_trace` HARD gate).""" + assert len(REPORT_TRIAGER_TOOLS) == 1 + assert REPORT_TRIAGER_TOOLS[0] is read + for forbidden in (bash, edit, gh, webfetch): + assert forbidden not in REPORT_TRIAGER_TOOLS, ( + f"{forbidden.__name__} must not be in the report triager's tools" + ) + + +def test_report_triager_builds_with_triage_output() -> None: + """It can be constructed (the executor builds it per-run) and emits the + shared TriageOutput contract.""" + from pydantic_ai.models.test import TestModel + + agent = build_agent(TestModel()) + assert agent is not None + assert AGENT_OUTPUT_SCHEMAS["report_triager"] is TriageOutput + + +def test_report_import_is_always_tagged_report() -> None: + """A report import is force-tagged source_type='report' so triage routes + it to the report triager — even if the model guessed a scanner name.""" + assert REPORT_SOURCE_TYPE == "report" + assert _resolve_source_type("report", None) == "report" + assert _resolve_source_type("report", "snyk") == "report" + + +def test_scanner_source_type_preserved() -> None: + assert _resolve_source_type("snyk", "snyk") == "snyk" + assert _resolve_source_type("snyk", None) == "snyk" + assert _resolve_source_type("trivy", "trivy-secret") == "trivy-secret" From 869ae97583f7fc78c879202c165d6f25d248210c Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 19:52:36 +0300 Subject: [PATCH 04/11] feat(triage): M4 side panel triage sections + stage chips (PRD-0008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IssueStageChip: triaging (cyan pulse), triage_verdict ("Review verdict", awaiting tone), unexploitable (green, shield icon — distinct from false_positive's report icon so verdict = colour + icon + label). Fixed 3 stale Cyberdeck-migration tests in this file (cd-loader, 9.5px sm). - IssueSidePanel: a `triage` section (SPTriage) — verdict banner (colour + icon + label + confidence as word + %), progressive-disclosure ProofChecks (
), reachability graph incl. the calm "No path found" state, report claim-vs-code side-by-side, and an editable drafted reply that is never auto-sent. Verdict-dependent footer (real → Open workspace to remediate; needs_review → remediate/close; close → two-way ClosePicker with radio semantics, pre-selecting recommended_close). Triage closes reopen to `new` (untriaged). New sections use cd-* tokens (per the panel gotcha). - api/client.ts already carries the contract types (M1). - 8 new panel tests: all four verdict banners, the no-path state, the real footer, the two-way picker, the unexploitable close, report claim-compare + editable-not-auto-sent reply, and Reopen on a triage close. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/components/issues/IssueSidePanel.tsx | 588 +++++++++++++++++- .../src/components/issues/IssueStageChip.tsx | 17 +- .../issues/__tests__/IssueSidePanel.test.tsx | 195 ++++++ .../issues/__tests__/IssueStageChip.test.tsx | 46 +- 4 files changed, 828 insertions(+), 18 deletions(-) diff --git a/frontend/src/components/issues/IssueSidePanel.tsx b/frontend/src/components/issues/IssueSidePanel.tsx index 92489902..ec43292a 100644 --- a/frontend/src/components/issues/IssueSidePanel.tsx +++ b/frontend/src/components/issues/IssueSidePanel.tsx @@ -19,7 +19,15 @@ */ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { parseApiError } from '../../api/client' -import type { AgentRun, ExceptionReason, Finding, IssueStage } from '../../api/client' +import type { + AgentRun, + ExceptionReason, + Finding, + IssueStage, + TriageCheck, + TriageOutput, + TriageVerdict, +} from '../../api/client' import { useAgentRuns, useApprovePlan, @@ -49,15 +57,33 @@ interface IssueSidePanelProps { starting?: boolean } -type SectionKey = 'plan' | 'plan_drafting' | 'pr' | 'validation' | 'finding' | 'activity' +type SectionKey = + | 'plan' + | 'plan_drafting' + | 'pr' + | 'validation' + | 'finding' + | 'activity' + // ADR-0051 / PRD-0008 — the triage verdict + proof. + | 'triage' const REASON_OPTIONS: { value: ExceptionReason; label: string }[] = [ { value: 'false_positive', label: 'False positive' }, + // ADR-0051 §7 — a real advisory that isn't reachable/exploitable here. + { value: 'unexploitable', label: 'Unexploitable' }, { value: 'wont_fix', label: "Won't fix" }, { value: 'accepted_risk', label: 'Accept risk' }, { value: 'deferred', label: 'Defer' }, ] +// ADR-0051 §7 — the two-way triage close: a real advisory that can't reach the +// user (unexploitable, recommended for the dependency flood) vs not a real +// issue (false_positive). Ordered with unexploitable first per UX-0008. +const TRIAGE_CLOSE_OPTIONS: { value: ExceptionReason; label: string }[] = [ + { value: 'unexploitable', label: 'Unexploitable' }, + { value: 'false_positive', label: 'False positive' }, +] + function severityKind(raw: string | null): IssueSeverityKind { const key = (raw ?? 'medium').toLowerCase() if (key === 'critical' || key === 'high' || key === 'low') return key @@ -86,15 +112,27 @@ function sectionsForStage(stage: IssueStage): SectionKey[] { if (stage === 'pr_ready' || stage === 'pr_awaiting_val') { return ['pr', 'plan', 'activity', 'finding'] } + // ADR-0051 / PRD-0008 — triage closes (false_positive / unexploitable) were + // resolved at the triage gate, before any remediation: surface the triage + // evidence that closed them (auditable + reopenable, Story 6), not an + // empty plan/PR/validation block. + if (stage === 'false_positive' || stage === 'unexploitable') { + return ['triage', 'activity', 'finding'] + } if ( stage === 'fixed' || - stage === 'false_positive' || stage === 'wont_fix' || stage === 'accepted' || stage === 'deferred' ) { return ['validation', 'pr', 'plan', 'activity', 'finding'] } + // ADR-0051 — triage in flight or a verdict awaiting confirmation: the + // verdict + progressive-disclosure proof lead; the live agent history and + // finding metadata support it. + if (stage === 'triaging' || stage === 'triage_verdict') { + return ['triage', 'activity', 'finding'] + } if ( stage === 'planning' || stage === 'generating' || @@ -211,6 +249,8 @@ export function IssueSidePanel({ ) if (key === 'plan_drafting') return + if (key === 'triage') + return if (key === 'pr') return if (key === 'validation') return @@ -583,6 +623,377 @@ function SPPullRequest({ prUrl }: { prUrl: string | null }) { ) } +// --------------------------------------------------------------------------- +// Triage section (ADR-0051 / PRD-0008 / UX-0008) +// --------------------------------------------------------------------------- + +/** Render confidence as a word + % ("High · 92%") — never a bare number, + * never stars (UX-0008 §States). */ +function confidenceText(confidence: number): string { + const pct = Math.round(confidence * 100) + const word = confidence >= 0.85 ? 'High' : confidence >= 0.7 ? 'Medium' : 'Low' + return `${word} · ${pct}%` +} + +interface VerdictVisual { + label: string + icon: string + color: string + /** Background tint token for the banner. */ + tint: string +} + +// Verdict = colour + icon + label, never colour alone (UX-0008 §States, +// DESIGN.md severity rule). unexploitable/false_positive carry DISTINCT icons. +const VERDICT_VISUAL: Record = { + real: { + label: 'Real risk', + icon: 'warning', + color: 'var(--cd-red, #ef6464)', + tint: 'var(--cd-red-dim, rgba(239,100,100,0.12))', + }, + unexploitable: { + label: 'Not exploitable', + icon: 'shield', + color: 'var(--cd-green)', + tint: 'var(--cd-green-dim, rgba(122,200,160,0.12))', + }, + false_positive: { + label: 'False positive', + icon: 'report', + color: 'var(--cd-green)', + tint: 'var(--cd-green-dim, rgba(122,200,160,0.12))', + }, + needs_review: { + label: 'Needs your review', + icon: 'help', + color: 'var(--cd-amber, #f5b54a)', + tint: 'var(--cd-amber-dim, rgba(245,181,74,0.12))', + }, +} + +const CHECK_ICON: Record = { + pass: 'check_circle', + fail: 'cancel', + warn: 'help', + info: 'info', +} + +const CHECK_COLOR: Record = { + pass: 'var(--cd-green)', + fail: 'var(--cd-red, #ef6464)', + warn: 'var(--cd-amber, #f5b54a)', + info: 'var(--cd-fg-3)', +} + +// The live step list shown while scanner triage reasons (UX-0008 Story 1). +const TRIAGE_STEPS = [ + 'Reading the advisory', + 'Mapping your call graph', + 'Checking reachability', + 'Tracing untrusted input', + 'Weighing the verdict', +] + +function TriageProgress() { + return ( +
+ +
    + {TRIAGE_STEPS.map((s) => ( +
  • + {s} +
    • + ))} +
+
+ ) +} + +function VerdictBanner({ triage }: { triage: TriageOutput }) { + const v = VERDICT_VISUAL[triage.verdict] ?? VERDICT_VISUAL.needs_review + const rationale = + triage.exploitability?.reason ?? triage.reachability?.summary ?? null + return ( +
+
+ + {v.icon} + + + {v.label} + + + {confidenceText(triage.confidence)} + +
+ {rationale && ( +

+ {rationale} +

+ )} +
+ ) +} + +function ProofChecks({ checks }: { checks: TriageCheck[] }) { + if (checks.length === 0) return null + return ( +
+ + Cliff checked {checks.length} thing{checks.length === 1 ? '' : 's'} + +
    + {checks.map((c, i) => ( +
  • + + {CHECK_ICON[c.kind] ?? 'info'} + +
    +
    + + {c.eyebrow} + + + {c.result} + +
    + {c.detail && ( +

    + {c.detail} +

    + )} +
    +
    • + ))} +
+
+ ) +} + +function ReachabilityGraph({ triage }: { triage: TriageOutput }) { + const r = triage.reachability + if (!r) return null + // The calm "No path found" state — the most reassuring picture on the + // screen (PRD-0008 Story 2). No alarmist styling. + if (!r.reached) { + return ( +
+ + Reachability + +
+ + do_not_disturb_on + + + No path found + + {r.summary && ( + + — {r.summary} + + )} +
+
+ ) + } + return ( +
+ + Reachability + +
+ {(r.path.length > 0 + ? r.path + : [{ label: r.summary ?? 'Reachable', detail: null, kind: null }] + ).map((node, i, arr) => ( +
+
+ + {i < arr.length - 1 && ( + + )} +
+
+
+ {node.label} +
+ {node.detail && ( +
+ {node.detail} +
+ )} +
+
+ ))} +
+
+ ) +} + +function ClaimCompare({ triage }: { triage: TriageOutput }) { + const cmp = triage.report?.claim_vs_code + if (!cmp) return null + return ( +
+ + Claim vs your code{cmp.file ? ` — ${cmp.file}` : ''} + +
+
+
+ Reporter’s claim +
+ 
+            {cmp.claimed ?? '—'}
+
+
+
+
+ Your code +
+ 
+            {cmp.actual ?? '—'}
+
+
+
+ {cmp.assessment && ( +

+ {cmp.assessment} +

+ )} +
+ ) +} + +function DraftedReply({ triage }: { triage: TriageOutput }) { + const draft = triage.report?.drafted_reply + const [text, setText] = useState(draft ?? '') + if (!draft) return null + return ( +
+
+ Drafted reply + + you edit and send this — Cliff never sends it for you + +
+