From ddd02bc13c9a00886be3948889a730127ccd217d Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 21:58:54 +0300 Subject: [PATCH 01/34] feat(repos): repo entity + canonicalizer + per-repo profile store (ADR-0053 P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IMPL-0025 Phase 1 foundation: first-class repo entity (migration 025) keyed by a canonicalized URL (the normalization the app lacked), its DAO with a one-build-per-repo CAS mutex + stale-build reaper, and RepoDirManager — the shared per-repo Project-profile store with atomic writes and a lazy generated PROFILE.md. 49 tests, TDD-first. Co-Authored-By: Claude Opus 4.8 --- .../cliff/db/migrations/025_repo_entity.sql | 25 +++ backend/cliff/models/__init__.py | 7 + backend/cliff/models/repo.py | 34 ++++ backend/cliff/repos/__init__.py | 7 + backend/cliff/repos/dao.py | 176 +++++++++++++++++ backend/cliff/repos/identity.py | 56 ++++++ backend/cliff/repos/repo_dir_manager.py | 177 ++++++++++++++++++ backend/tests/repos/__init__.py | 0 backend/tests/repos/conftest.py | 17 ++ backend/tests/repos/test_identity.py | 65 +++++++ backend/tests/repos/test_repo_dao.py | 129 +++++++++++++ backend/tests/repos/test_repo_dir_manager.py | 108 +++++++++++ 12 files changed, 801 insertions(+) create mode 100644 backend/cliff/db/migrations/025_repo_entity.sql create mode 100644 backend/cliff/models/repo.py create mode 100644 backend/cliff/repos/__init__.py create mode 100644 backend/cliff/repos/dao.py create mode 100644 backend/cliff/repos/identity.py create mode 100644 backend/cliff/repos/repo_dir_manager.py create mode 100644 backend/tests/repos/__init__.py create mode 100644 backend/tests/repos/conftest.py create mode 100644 backend/tests/repos/test_identity.py create mode 100644 backend/tests/repos/test_repo_dao.py create mode 100644 backend/tests/repos/test_repo_dir_manager.py diff --git a/backend/cliff/db/migrations/025_repo_entity.sql b/backend/cliff/db/migrations/025_repo_entity.sql new file mode 100644 index 00000000..7c245f07 --- /dev/null +++ b/backend/cliff/db/migrations/025_repo_entity.sql @@ -0,0 +1,25 @@ +-- 025_repo_entity.sql — ADR-0053: first-class git-repository entity. +-- +-- Repo identity was a raw URL string duplicated across assessment/workspace/ +-- integration_config with no normalization. This table is the canonical home, +-- keyed by canonical_url (cliff.repos.identity.canonicalize_repo_url), and the +-- queryable store for Project-profile freshness. +-- +-- One-build-per-repo (ADR-0053 §6) is enforced by a compare-and-swap on +-- profile_status (DAO try_begin_profile), NOT a partial unique index: because +-- canonical_url is globally UNIQUE there is exactly one row per repo, so a +-- partial unique index on canonical_url WHERE building would be strictly +-- redundant with the UNIQUE constraint. The CAS is the honest mutex here. + +CREATE TABLE IF NOT EXISTS repo ( + id TEXT PRIMARY KEY, + canonical_url TEXT NOT NULL UNIQUE, + default_branch TEXT, + last_profiled_sha TEXT, + profiled_at TEXT, + profile_status TEXT NOT NULL DEFAULT 'none' + CHECK (profile_status IN ('none', 'building', 'ready', 'stale', 'error')), + profile_dir TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); diff --git a/backend/cliff/models/__init__.py b/backend/cliff/models/__init__.py index f918d8bb..3e2dd900 100644 --- a/backend/cliff/models/__init__.py +++ b/backend/cliff/models/__init__.py @@ -53,6 +53,10 @@ PostureCheckName, PostureCheckStatus, ) +from cliff.models.repo import ( + Repo, + RepoProfileStatus, +) # --------------------------------------------------------------------------- # Workspace @@ -540,6 +544,9 @@ class IngestResult(BaseModel): "PostureCheckCategory", "PostureCheckName", "PostureCheckStatus", + # Repo (ADR-0053) + "Repo", + "RepoProfileStatus", # Completion "Completion", "CompletionCreate", diff --git a/backend/cliff/models/repo.py b/backend/cliff/models/repo.py new file mode 100644 index 00000000..76aaf26c --- /dev/null +++ b/backend/cliff/models/repo.py @@ -0,0 +1,34 @@ +"""Repo entity (ADR-0053). + +The first-class git-repository in Cliff. Its ``canonical_url`` (via +:func:`cliff.repos.identity.canonicalize_repo_url`) is the de-duplicating key, +and it is the queryable home for the Project-profile freshness the dashboard +shows. The profile artifacts themselves live on the filesystem under +``profile_dir``; this row is the metadata + freshness pointer (the hybrid +SQLite-metadata + filesystem-blob split, mirroring ``workspace`` + +``workspace_dir``). +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TCH003 — Pydantic needs this at runtime +from typing import Literal + +from pydantic import BaseModel + +#: ``none`` — never profiled. ``building`` — a profile build holds the mutex. +#: ``ready`` — a fresh profile exists. ``stale`` — code moved past +#: ``last_profiled_sha``. ``error`` — the last build failed. +RepoProfileStatus = Literal["none", "building", "ready", "stale", "error"] + + +class Repo(BaseModel): + id: str + canonical_url: str + default_branch: str | None = None + last_profiled_sha: str | None = None + profiled_at: datetime | None = None + profile_status: RepoProfileStatus = "none" + profile_dir: str | None = None + created_at: datetime + updated_at: datetime diff --git a/backend/cliff/repos/__init__.py b/backend/cliff/repos/__init__.py new file mode 100644 index 00000000..1af0f0d8 --- /dev/null +++ b/backend/cliff/repos/__init__.py @@ -0,0 +1,7 @@ +"""Repo knowledge base (ADR-0053). + +A first-class git-repository entity plus the per-repo "Project profile" store +that the agentic triage Deep dive (ADR-0052) reads. Distinct from the +``cliff.db.repo_*`` modules, which are the data-access ("repository pattern") +layer for other entities — this package is the git-repository feature itself. +""" diff --git a/backend/cliff/repos/dao.py b/backend/cliff/repos/dao.py new file mode 100644 index 00000000..081f8824 --- /dev/null +++ b/backend/cliff/repos/dao.py @@ -0,0 +1,176 @@ +"""Data access for the ``repo`` entity (ADR-0053). + +Kept in the feature package (not ``cliff.db.repo_*``) to avoid the naming +collision between the git-repository entity and the data-access +("repository pattern") modules. Functions take an ``aiosqlite.Connection`` +like the rest of the DAO layer. + +All timestamps are written as Python UTC isoformat strings so ordering +comparisons (the stale-build reaper) are consistent — the table's +``datetime('now')`` defaults are a fallback we don't rely on. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +from cliff.models.repo import Repo, RepoProfileStatus +from cliff.repos.identity import canonicalize_repo_url + +if TYPE_CHECKING: + import aiosqlite + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +def _row_to_repo(row: aiosqlite.Row) -> Repo: + return Repo( + id=row["id"], + canonical_url=row["canonical_url"], + default_branch=row["default_branch"], + last_profiled_sha=row["last_profiled_sha"], + profiled_at=row["profiled_at"], + profile_status=row["profile_status"], + profile_dir=row["profile_dir"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +async def get_or_create_repo(db: aiosqlite.Connection, repo_url: str) -> Repo: + """Return the repo for *repo_url*, creating it on first sight. + + The URL is canonicalized first, so every spelling of the same repository + resolves to one row. + """ + canonical = canonicalize_repo_url(repo_url) + now = _now() + await db.execute( + """ + INSERT INTO repo (id, canonical_url, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(canonical_url) DO NOTHING + """, + (uuid.uuid4().hex, canonical, now, now), + ) + await db.commit() + repo = await _get_by_canonical(db, canonical) + assert repo is not None # just inserted or already present + return repo + + +async def _get_by_canonical( + db: aiosqlite.Connection, canonical: str +) -> Repo | None: + cursor = await db.execute( + "SELECT * FROM repo WHERE canonical_url = ?", (canonical,) + ) + row = await cursor.fetchone() + return _row_to_repo(row) if row else None + + +async def get_repo(db: aiosqlite.Connection, repo_id: str) -> Repo | None: + cursor = await db.execute("SELECT * FROM repo WHERE id = ?", (repo_id,)) + row = await cursor.fetchone() + return _row_to_repo(row) if row else None + + +async def get_repo_by_url(db: aiosqlite.Connection, repo_url: str) -> Repo | None: + """Look up a repo by any spelling of its URL (canonicalized first).""" + return await _get_by_canonical(db, canonicalize_repo_url(repo_url)) + + +async def list_repos(db: aiosqlite.Connection) -> list[Repo]: + cursor = await db.execute("SELECT * FROM repo ORDER BY canonical_url") + return [_row_to_repo(row) for row in await cursor.fetchall()] + + +async def try_begin_profile(db: aiosqlite.Connection, repo_id: str) -> bool: + """Acquire the one-build-per-repo mutex (ADR-0053 §6). + + Compare-and-swap on ``profile_status``: flips the single repo row to + ``building`` only if it is not already building. Returns ``True`` when this + caller acquired the build, ``False`` when a build is already in progress + (or the repo doesn't exist). + """ + cursor = await db.execute( + """ + UPDATE repo + SET profile_status = 'building', updated_at = ? + WHERE id = ? AND profile_status != 'building' + """, + (_now(), repo_id), + ) + await db.commit() + return cursor.rowcount == 1 + + +async def finish_profile( + db: aiosqlite.Connection, + repo_id: str, + *, + status: RepoProfileStatus, + sha: str | None = None, + profile_dir: str | None = None, +) -> Repo | None: + """Release the build mutex with a terminal *status*. + + On ``ready`` the freshness fields (``last_profiled_sha``, ``profiled_at``, + ``profile_dir``) are stamped; on ``error`` only the status changes so the + previous good profile's freshness survives. + """ + now = _now() + if status == "ready": + await db.execute( + """ + UPDATE repo + SET profile_status = 'ready', + last_profiled_sha = COALESCE(?, last_profiled_sha), + profiled_at = ?, + profile_dir = COALESCE(?, profile_dir), + updated_at = ? + WHERE id = ? + """, + (sha, now, profile_dir, now, repo_id), + ) + else: + await db.execute( + "UPDATE repo SET profile_status = ?, updated_at = ? WHERE id = ?", + (status, now, repo_id), + ) + await db.commit() + return await get_repo(db, repo_id) + + +async def mark_stale(db: aiosqlite.Connection, repo_id: str) -> Repo | None: + await db.execute( + "UPDATE repo SET profile_status = 'stale', updated_at = ? WHERE id = ?", + (_now(), repo_id), + ) + await db.commit() + return await get_repo(db, repo_id) + + +async def reap_stale_builds( + db: aiosqlite.Connection, *, older_than_seconds: float +) -> int: + """Mark profile builds that have been ``building`` past the threshold as + ``error`` (watchdog — the build task died without releasing the mutex). + + Returns the number of rows reaped. + """ + cutoff = (datetime.now(UTC) - timedelta(seconds=older_than_seconds)).isoformat() + cursor = await db.execute( + """ + UPDATE repo + SET profile_status = 'error', updated_at = ? + WHERE profile_status = 'building' AND updated_at < ? + """, + (_now(), cutoff), + ) + await db.commit() + return cursor.rowcount diff --git a/backend/cliff/repos/identity.py b/backend/cliff/repos/identity.py new file mode 100644 index 00000000..860968ac --- /dev/null +++ b/backend/cliff/repos/identity.py @@ -0,0 +1,56 @@ +"""Repository identity — canonicalize git remote URLs to a stable key. + +ADR-0053 §1: the app had no URL normalization, so ``…/a/b``, ``…/a/b.git`` and +``git@host:a/b`` were three different repos to the existing code (which used the +raw URL string as a de-facto key in ``assessment``/``workspace``/ +``integration_config``). ``canonicalize_repo_url`` collapses the common +spellings to one ``https:////`` form used as +``repo.canonical_url``. +""" + +from __future__ import annotations + +import re +from urllib.parse import urlparse + +# scp-like git syntax: ``[user@]host:owner/repo`` (no scheme, a single colon +# before the path). Distinguished from a real URL by the absence of ``://``. +_SCP_LIKE = re.compile(r"^(?:[^@/]+@)?(?P[^:/]+):(?P.+)$") + +_GIT_SUFFIX = ".git" + + +class InvalidRepoUrlError(ValueError): + """The string could not be parsed as a usable repository URL.""" + + +def canonicalize_repo_url(raw: str) -> str: + """Return the canonical ``https:////`` key for *raw*. + + Normalizes scheme to https, lowercases the host, strips embedded + credentials, a trailing ``.git``, and trailing slashes. The path case is + preserved (some hosts are case-sensitive). Raises :class:`InvalidRepoUrlError` + for empty input or a URL without both a host and an owner/repo path. + """ + if not raw or not raw.strip(): + raise InvalidRepoUrlError("repo url must not be empty") + s = raw.strip() + + # scp-like or bare host/path → give it an https scheme so urlparse works. + if "://" not in s: + scp = _SCP_LIKE.match(s) + s = f"https://{scp.group('host')}/{scp.group('path')}" if scp else f"https://{s}" + + parsed = urlparse(s) + host = (parsed.hostname or "").lower() + if not host: + raise InvalidRepoUrlError(f"repo url has no host: {raw!r}") + + path = parsed.path.strip("/") + if path.endswith(_GIT_SUFFIX): + path = path[: -len(_GIT_SUFFIX)] + path = path.strip("/") + if not path: + raise InvalidRepoUrlError(f"repo url has no owner/repo path: {raw!r}") + + return f"https://{host}/{path}" diff --git a/backend/cliff/repos/repo_dir_manager.py b/backend/cliff/repos/repo_dir_manager.py new file mode 100644 index 00000000..4c9412b8 --- /dev/null +++ b/backend/cliff/repos/repo_dir_manager.py @@ -0,0 +1,177 @@ +"""RepoDirManager — the per-repo Project-profile store on disk (ADR-0053 §2). + +Mirrors ``WorkspaceDirManager`` (the per-*finding* runtime) for the per-*repo* +tier, with two deliberate differences the shared store needs: + +* **Atomic writes** (tmp + ``os.replace``). The workspace runtime relies on a + one-writer-per-directory invariant; the repo store is read by many concurrent + triage runs while the profiler writes, so a reader must never see a + half-written file — it gets the previous committed version instead. +* **Lazy ``PROFILE.md``.** The human/agent digest is regenerated on + profile-complete via :meth:`regenerate_profile_md`, **not** on every artifact + write (``code_map.json`` for a monorepo can be large, unlike the tiny + workspace sections). + +Source of truth is the JSON artifacts; ``PROFILE.md`` is generated from them. +""" + +from __future__ import annotations + +import json +import os +import shutil +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +#: Bump when an artifact's on-disk shape changes incompatibly. A mismatch means +#: "stale — rebuild," never "migrate" (ADR-0053 §5): the artifacts are +#: regenerable derivatives. +SCHEMA_VERSION = 1 + +#: The Project-profile artifacts (ADR-0053 §3). The clone lives alongside in +#: ``repo/`` (see :meth:`clone_dir`). +ARTIFACTS = ("profile", "code_map", "threat") + + +class RepoDirManager: + """Creates and serves per-repo store directories under ``base_dir``.""" + + def __init__(self, base_dir: Path) -> None: + self._base_dir = base_dir + + @property + def base_dir(self) -> Path: + return self._base_dir + + # -- paths -------------------------------------------------------------- + + def _repo_root(self, repo_id: str) -> Path: + _validate_repo_id(repo_id) + return self._base_dir / repo_id + + def repo_dir(self, repo_id: str) -> str: + """The store directory path, as a string for ``repo.profile_dir``.""" + return str(self._repo_root(repo_id)) + + def clone_dir(self, repo_id: str) -> Path: + """The one cached clone for this repo (ADR-0052 §3 reads it read-only).""" + return self._repo_root(repo_id) / "repo" + + def profile_md_path(self, repo_id: str) -> Path: + return self._repo_root(repo_id) / "PROFILE.md" + + def ensure(self, repo_id: str) -> Path: + root = self._repo_root(repo_id) + root.mkdir(parents=True, exist_ok=True) + return root + + # -- artifacts ---------------------------------------------------------- + + def write_artifact(self, repo_id: str, name: str, data: dict) -> None: + if name not in ARTIFACTS: + raise ValueError( + f"Unknown profile artifact {name!r}; expected one of {ARTIFACTS}" + ) + root = self.ensure(repo_id) + _atomic_write_json(root / f"{name}.json", data) + + def read_artifact(self, repo_id: str, name: str) -> dict | None: + path = self._repo_root(repo_id) / f"{name}.json" + if not path.exists(): + return None + return json.loads(path.read_text()) + + # -- manifest ----------------------------------------------------------- + + def write_manifest( + self, + repo_id: str, + *, + source_sha: str | None = None, + built_at: str | None = None, + ) -> dict: + """Write the index of what the store holds and how fresh it is.""" + root = self.ensure(repo_id) + present = [n for n in ARTIFACTS if (root / f"{n}.json").exists()] + manifest = { + "schema_version": SCHEMA_VERSION, + "source_sha": source_sha, + "built_at": built_at, + "artifacts": present, + } + _atomic_write_json(root / "MANIFEST.json", manifest) + return manifest + + def read_manifest(self, repo_id: str) -> dict | None: + path = self._repo_root(repo_id) / "MANIFEST.json" + if not path.exists(): + return None + return json.loads(path.read_text()) + + # -- generated digest --------------------------------------------------- + + def regenerate_profile_md(self, repo_id: str) -> None: + """Rebuild ``PROFILE.md`` from the current artifacts (call on complete).""" + root = self.ensure(repo_id) + sections = {name: self.read_artifact(repo_id, name) for name in ARTIFACTS} + _atomic_write_text(root / "PROFILE.md", _render_profile_md(sections)) + + # -- delete ------------------------------------------------------------- + + def delete(self, repo_id: str) -> bool: + root = self._repo_root(repo_id) + if not root.exists(): + return False + shutil.rmtree(root) + return True + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _atomic_write_text(path: Path, text: str) -> None: + """Write *text* atomically — a concurrent reader sees old-or-new, never torn.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(text) + os.replace(tmp, path) + + +def _atomic_write_json(path: Path, data: dict) -> None: + _atomic_write_text(path, json.dumps(data, indent=2) + "\n") + + +def _render_profile_md(sections: dict[str, dict | None]) -> str: + """Render a readable digest from the artifacts (the source of truth). + + Deliberately generic: it renders the scalar top-level fields of whatever + artifacts are present. The profile builders (later) define the exact shapes; + this stays robust to that without coupling to a schema that doesn't exist + yet. + """ + lines = ["# Project profile", ""] + present = {name: data for name, data in sections.items() if data} + if not present: + lines.append("_No profile built yet._") + return "\n".join(lines) + "\n" + for name, data in present.items(): + lines.append(f"## {name.replace('_', ' ')}") + lines.append("") + for key, value in data.items(): + if isinstance(value, (str, int, float, bool)): + lines.append(f"- **{key}:** {value}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def _validate_repo_id(repo_id: str) -> None: + if not repo_id: + raise ValueError("repo_id must not be empty") + if "/" in repo_id or "\\" in repo_id: + raise ValueError(f"repo_id must not contain path separators: {repo_id!r}") + if repo_id in (".", ".."): + raise ValueError(f"repo_id must not be a relative path component: {repo_id!r}") diff --git a/backend/tests/repos/__init__.py b/backend/tests/repos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/repos/conftest.py b/backend/tests/repos/conftest.py new file mode 100644 index 00000000..b814460b --- /dev/null +++ b/backend/tests/repos/conftest.py @@ -0,0 +1,17 @@ +"""Fixtures for repo-knowledge-base tests — in-memory SQLite with migrations.""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +async def db(): + """An aiosqlite.Connection backed by an in-memory DB with all migrations run.""" + from cliff.db.connection import close_db, init_db + + conn = await init_db(":memory:") + try: + yield conn + finally: + await close_db() diff --git a/backend/tests/repos/test_identity.py b/backend/tests/repos/test_identity.py new file mode 100644 index 00000000..1382eb2e --- /dev/null +++ b/backend/tests/repos/test_identity.py @@ -0,0 +1,65 @@ +"""Unit tests for repo URL canonicalization (ADR-0053 §1). + +The app had no URL normalization, so the same repository spelled three ways was +three repos to the existing code. ``canonicalize_repo_url`` collapses the common +spellings to one ``https:////`` key. +""" + +from __future__ import annotations + +import pytest + +from cliff.repos.identity import InvalidRepoUrlError, canonicalize_repo_url + +CANONICAL = "https://github.com/acme/web" + + +@pytest.mark.parametrize( + "raw", + [ + "https://github.com/acme/web", + "https://github.com/acme/web.git", + "https://github.com/acme/web/", + "https://github.com/acme/web/.git", # trailing slash + .git + "http://github.com/acme/web", # scheme normalised to https + "https://GitHub.com/acme/web", # host lowercased + "https://github.com/acme/web ", # surrounding whitespace + "git@github.com:acme/web.git", # scp-like + "git@github.com:acme/web", + "ssh://git@github.com/acme/web.git", # ssh scheme + "https://x-access-token:ghp_secret@github.com/acme/web.git", # creds stripped + "github.com/acme/web", # bare, no scheme + ], +) +def test_spellings_collapse_to_one_key(raw): + assert canonicalize_repo_url(raw) == CANONICAL + + +def test_path_case_is_preserved(): + # GitHub is case-insensitive for access but other hosts are not; only the + # host is lowercased, the path is left intact. + assert canonicalize_repo_url("https://gitlab.com/Group/Sub/Repo") == ( + "https://gitlab.com/Group/Sub/Repo" + ) + + +def test_non_github_host_preserved(): + assert ( + canonicalize_repo_url("git@gitlab.example.com:team/svc.git") + == "https://gitlab.example.com/team/svc" + ) + + +def test_token_never_survives_in_key(): + key = canonicalize_repo_url("https://x-access-token:ghp_secret@github.com/a/b") + assert "ghp_secret" not in key + assert "x-access-token" not in key + + +@pytest.mark.parametrize( + "bad", + ["", " ", "https://github.com", "https://github.com/", "not a url at all/"], +) +def test_rejects_unusable(bad): + with pytest.raises(InvalidRepoUrlError): + canonicalize_repo_url(bad) diff --git a/backend/tests/repos/test_repo_dao.py b/backend/tests/repos/test_repo_dao.py new file mode 100644 index 00000000..338ff542 --- /dev/null +++ b/backend/tests/repos/test_repo_dao.py @@ -0,0 +1,129 @@ +"""Unit tests for the repo entity DAO + migration 025 (ADR-0053).""" + +from __future__ import annotations + +import asyncio +import sqlite3 + +import pytest + +from cliff.repos.dao import ( + finish_profile, + get_or_create_repo, + get_repo, + get_repo_by_url, + list_repos, + reap_stale_builds, + try_begin_profile, +) + +A = "https://github.com/acme/web" + + +async def test_get_or_create_dedups_across_spellings(db): + one = await get_or_create_repo(db, A) + two = await get_or_create_repo(db, "git@github.com:acme/web.git") + three = await get_or_create_repo(db, "https://github.com/acme/web/") + assert one.id == two.id == three.id + assert one.canonical_url == A + assert len(await list_repos(db)) == 1 + + +async def test_new_repo_defaults(db): + repo = await get_or_create_repo(db, A) + assert repo.profile_status == "none" + assert repo.profiled_at is None + assert repo.profile_dir is None + assert repo.last_profiled_sha is None + assert repo.created_at is not None + + +async def test_get_repo_by_url_finds_via_variant(db): + created = await get_or_create_repo(db, A) + found = await get_repo_by_url(db, "HTTPS://GitHub.com/acme/web.git") + assert found is not None + assert found.id == created.id + + +async def test_get_repo_missing(db): + assert await get_repo(db, "nope") is None + + +async def test_canonical_url_is_unique(db): + await get_or_create_repo(db, A) + with pytest.raises(sqlite3.IntegrityError): + await db.execute( + "INSERT INTO repo (id, canonical_url, created_at, updated_at) VALUES (?,?,?,?)", + ("dup", A, "2026-01-01T00:00:00+00:00", "2026-01-01T00:00:00+00:00"), + ) + await db.commit() + + +# ── the one-build-per-repo mutex (CAS) ────────────────────────────────────── + + +async def test_try_begin_profile_is_a_mutex(db): + repo = await get_or_create_repo(db, A) + assert await try_begin_profile(db, repo.id) is True + # A second attempt while building is rejected — the mutex is held. + assert await try_begin_profile(db, repo.id) is False + + refreshed = await get_repo(db, repo.id) + assert refreshed.profile_status == "building" + + +async def test_finish_profile_ready_stamps_freshness_and_releases(db): + repo = await get_or_create_repo(db, A) + await try_begin_profile(db, repo.id) + done = await finish_profile( + db, repo.id, status="ready", sha="abc123", profile_dir="data/repos/x" + ) + assert done.profile_status == "ready" + assert done.last_profiled_sha == "abc123" + assert done.profile_dir == "data/repos/x" + assert done.profiled_at is not None + # Mutex released — a rebuild can be acquired again. + assert await try_begin_profile(db, repo.id) is True + + +async def test_finish_profile_error_preserves_prior_freshness(db): + repo = await get_or_create_repo(db, A) + await try_begin_profile(db, repo.id) + await finish_profile(db, repo.id, status="ready", sha="good-sha", profile_dir="d") + # A later build fails. + await try_begin_profile(db, repo.id) + errored = await finish_profile(db, repo.id, status="error") + assert errored.profile_status == "error" + # The previous good profile's freshness survives an error. + assert errored.last_profiled_sha == "good-sha" + assert errored.profile_dir == "d" + + +async def test_try_begin_profile_missing_repo(db): + assert await try_begin_profile(db, "ghost") is False + + +# ── stale-build reaper (watchdog) ─────────────────────────────────────────── + + +async def test_reap_stale_builds_marks_old_building_rows_error(db): + repo = await get_or_create_repo(db, A) + await try_begin_profile(db, repo.id) + await asyncio.sleep(0.05) + reaped = await reap_stale_builds(db, older_than_seconds=0.001) + assert reaped == 1 + assert (await get_repo(db, repo.id)).profile_status == "error" + + +async def test_reap_stale_builds_skips_recent(db): + repo = await get_or_create_repo(db, A) + await try_begin_profile(db, repo.id) + reaped = await reap_stale_builds(db, older_than_seconds=60.0) + assert reaped == 0 + assert (await get_repo(db, repo.id)).profile_status == "building" + + +async def test_reap_stale_builds_ignores_non_building(db): + await get_or_create_repo(db, A) # status 'none' — never building + reaped = await reap_stale_builds(db, older_than_seconds=0.0) + assert reaped == 0 diff --git a/backend/tests/repos/test_repo_dir_manager.py b/backend/tests/repos/test_repo_dir_manager.py new file mode 100644 index 00000000..3822dfde --- /dev/null +++ b/backend/tests/repos/test_repo_dir_manager.py @@ -0,0 +1,108 @@ +"""Unit tests for the per-repo Project-profile store (ADR-0053 §2).""" + +from __future__ import annotations + +import pytest + +from cliff.repos.repo_dir_manager import SCHEMA_VERSION, RepoDirManager + +RID = "abc123def" + + +@pytest.fixture +def mgr(tmp_path): + return RepoDirManager(tmp_path / "repos") + + +def test_artifact_round_trip(mgr): + mgr.write_artifact(RID, "profile", {"kind": "service", "internet_facing": True}) + assert mgr.read_artifact(RID, "profile") == { + "kind": "service", + "internet_facing": True, + } + + +def test_read_missing_artifact_is_none(mgr): + assert mgr.read_artifact(RID, "profile") is None + + +def test_unknown_artifact_rejected(mgr): + with pytest.raises(ValueError, match="Unknown profile artifact"): + mgr.write_artifact(RID, "bogus", {}) + + +def test_write_is_atomic_no_tmp_left(mgr): + mgr.write_artifact(RID, "code_map", {"ships_roots": ["src/**"]}) + root = mgr.ensure(RID) + # No half-written temp file survives; the committed file parses. + assert not list(root.glob("*.tmp")) + assert mgr.read_artifact(RID, "code_map") == {"ships_roots": ["src/**"]} + + +def test_overwrite_replaces_cleanly(mgr): + mgr.write_artifact(RID, "threat", {"prior_issues": []}) + mgr.write_artifact(RID, "threat", {"prior_issues": [{"id": "CVE-1"}]}) + assert mgr.read_artifact(RID, "threat") == {"prior_issues": [{"id": "CVE-1"}]} + + +@pytest.mark.parametrize("bad", ["", "../escape", "a/b", "..", "x\\y"]) +def test_path_traversal_rejected(mgr, bad): + with pytest.raises(ValueError): + mgr.write_artifact(bad, "profile", {}) + + +def test_clone_dir_is_under_repo_root(mgr): + clone = mgr.clone_dir(RID) + assert clone.name == "repo" + assert clone.parent == mgr.ensure(RID) + + +# ── manifest ──────────────────────────────────────────────────────────────── + + +def test_manifest_lists_present_artifacts(mgr): + mgr.write_artifact(RID, "profile", {"kind": "library"}) + mgr.write_artifact(RID, "threat", {"prior_issues": []}) + manifest = mgr.write_manifest(RID, source_sha="deadbeef", built_at="2026-06-10T00:00:00Z") + assert manifest["schema_version"] == SCHEMA_VERSION + assert manifest["source_sha"] == "deadbeef" + assert set(manifest["artifacts"]) == {"profile", "threat"} + assert "code_map" not in manifest["artifacts"] + assert mgr.read_manifest(RID) == manifest + + +def test_read_manifest_missing_is_none(mgr): + assert mgr.read_manifest(RID) is None + + +# ── lazy PROFILE.md ─────────────────────────────────────────────────────────── + + +def test_profile_md_is_lazy_not_written_on_artifact_write(mgr): + mgr.write_artifact(RID, "profile", {"kind": "service"}) + # Writing an artifact must NOT regenerate the digest (it's lazy + can be big). + assert not mgr.profile_md_path(RID).exists() + + +def test_regenerate_profile_md_builds_readable_digest(mgr): + mgr.write_artifact(RID, "profile", {"kind": "self_hosted_app", "internet_facing": True}) + mgr.regenerate_profile_md(RID) + text = mgr.profile_md_path(RID).read_text() + assert text.startswith("# Project profile") + assert "self_hosted_app" in text + assert "internet_facing" in text + + +def test_regenerate_profile_md_empty_when_no_artifacts(mgr): + mgr.regenerate_profile_md(RID) + assert "No profile built yet" in mgr.profile_md_path(RID).read_text() + + +# ── delete ──────────────────────────────────────────────────────────────────── + + +def test_delete_removes_store(mgr): + mgr.write_artifact(RID, "profile", {"kind": "cli"}) + assert mgr.delete(RID) is True + assert mgr.read_artifact(RID, "profile") is None + assert mgr.delete(RID) is False From 684830e4a67ee4bf50ffdecaf54b842d2cc6baea Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 22:00:38 +0300 Subject: [PATCH 02/34] feat(repos): RepoKnowledge declared-consumption access layer (ADR-0053 P1.5) Agents declare which profile sections they consume; the loader populates only those, so no agent pays tokens for a section it doesn't use. Mirrors WorkspaceDeps for the per-repo tier. 5 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/repos/knowledge.py | 70 +++++++++++++++++++++++++++ backend/tests/repos/test_knowledge.py | 54 +++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 backend/cliff/repos/knowledge.py create mode 100644 backend/tests/repos/test_knowledge.py diff --git a/backend/cliff/repos/knowledge.py b/backend/cliff/repos/knowledge.py new file mode 100644 index 00000000..49d23c3f --- /dev/null +++ b/backend/cliff/repos/knowledge.py @@ -0,0 +1,70 @@ +"""RepoKnowledge — the declared-consumption access layer (ADR-0053 §5). + +Agents never open profile files directly. Each Tier-2 agent (ADR-0052) +*declares* which profile sections it consumes; the runtime loads **only** those +into a ``RepoKnowledge`` view and renders them into the prompt — so no agent +pays tokens for a section it doesn't use (the mechanism that makes eager +profiling affordable). Reading actual source is a separate concern: the +read/grep tools over :attr:`RepoKnowledge.clone_dir`. + +This mirrors how ``WorkspaceDeps`` already feeds per-finding context to agents, +for the per-repo tier. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from cliff.repos.repo_dir_manager import ARTIFACTS + +if TYPE_CHECKING: + from pathlib import Path + + from cliff.repos.repo_dir_manager import RepoDirManager + + +@dataclass(frozen=True) +class RepoKnowledge: + """A read-only, section-scoped view of one repo's Project profile. + + A section is ``None`` when the agent didn't declare it OR when it hasn't + been built yet — agents must treat ``None`` as "not available" and degrade, + never assume. + """ + + repo_id: str + profile: dict | None = None + code_map: dict | None = None + threat: dict | None = None + clone_dir: Path | None = None + + +def load_repo_knowledge( + mgr: RepoDirManager, + repo_id: str, + sections: list[str], + *, + include_clone: bool = False, +) -> RepoKnowledge: + """Load *only* the declared profile *sections* for *repo_id*. + + Args: + sections: which artifacts the consuming agent declared (subset of + ``ARTIFACTS``). Anything outside that set is a programming error. + include_clone: when True, expose the cached clone path for the + read/grep tools (``Trace the path``); otherwise left ``None``. + """ + selected = set(sections) + unknown = selected - set(ARTIFACTS) + if unknown: + raise ValueError( + f"Unknown profile section(s) {sorted(unknown)}; expected subset of {ARTIFACTS}" + ) + return RepoKnowledge( + repo_id=repo_id, + profile=mgr.read_artifact(repo_id, "profile") if "profile" in selected else None, + code_map=mgr.read_artifact(repo_id, "code_map") if "code_map" in selected else None, + threat=mgr.read_artifact(repo_id, "threat") if "threat" in selected else None, + clone_dir=mgr.clone_dir(repo_id) if include_clone else None, + ) diff --git a/backend/tests/repos/test_knowledge.py b/backend/tests/repos/test_knowledge.py new file mode 100644 index 00000000..bfdfe68a --- /dev/null +++ b/backend/tests/repos/test_knowledge.py @@ -0,0 +1,54 @@ +"""Unit tests for the RepoKnowledge declared-consumption access layer (ADR-0053 §5).""" + +from __future__ import annotations + +import pytest + +from cliff.repos.knowledge import load_repo_knowledge +from cliff.repos.repo_dir_manager import RepoDirManager + +RID = "repo123" + + +@pytest.fixture +def mgr(tmp_path): + m = RepoDirManager(tmp_path / "repos") + m.write_artifact(RID, "profile", {"kind": "service"}) + m.write_artifact(RID, "code_map", {"ships_roots": ["src/**"]}) + m.write_artifact(RID, "threat", {"prior_issues": []}) + return m + + +def test_loads_only_declared_sections(mgr): + k = load_repo_knowledge(mgr, RID, ["profile"]) + assert k.profile == {"kind": "service"} + # Undeclared sections are not loaded — the agent doesn't pay for them. + assert k.code_map is None + assert k.threat is None + + +def test_loads_all_declared(mgr): + k = load_repo_knowledge(mgr, RID, ["profile", "code_map", "threat"]) + assert k.profile and k.code_map and k.threat + + +def test_unknown_section_rejected(mgr): + with pytest.raises(ValueError, match="Unknown profile section"): + load_repo_knowledge(mgr, RID, ["profile", "bogus"]) + + +def test_declared_but_unbuilt_is_none(tmp_path): + empty = RepoDirManager(tmp_path / "repos") + k = load_repo_knowledge(empty, RID, ["profile", "code_map"]) + # Declared but never built → None; agents must degrade, not assume. + assert k.profile is None + assert k.code_map is None + + +def test_clone_dir_opt_in(mgr): + without = load_repo_knowledge(mgr, RID, ["profile"]) + assert without.clone_dir is None + + with_clone = load_repo_knowledge(mgr, RID, ["profile"], include_clone=True) + assert with_clone.clone_dir is not None + assert with_clone.clone_dir.name == "repo" From efa7ccd47cf6f610b8d7087e9510521cd8349bd7 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 22:24:26 +0300 Subject: [PATCH 03/34] feat(repos): credential-less cached clone + clone GC (ADR-0053 P1.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persistent clone obtains the token via GIT_ASKPASS+env (never argv/URL), so .git/config stays credential-less — verified by an offline clone against a local bare repo. refresh_repo fetches+resets to bring a clone current. gc_repo_clones evicts LRU clones over a byte budget while keeping the (re-clonable) profile artifacts. 10 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/repos/clone.py | 176 ++++++++++++++++++++++++++++++ backend/cliff/repos/gc.py | 53 +++++++++ backend/tests/repos/test_clone.py | 162 +++++++++++++++++++++++++++ 3 files changed, 391 insertions(+) create mode 100644 backend/cliff/repos/clone.py create mode 100644 backend/cliff/repos/gc.py create mode 100644 backend/tests/repos/test_clone.py diff --git a/backend/cliff/repos/clone.py b/backend/cliff/repos/clone.py new file mode 100644 index 00000000..709a3562 --- /dev/null +++ b/backend/cliff/repos/clone.py @@ -0,0 +1,176 @@ +"""Credential-less persistent clone for the per-repo store (ADR-0053 §4). + +The existing ``assessment.clone`` injects the GitHub token into the remote URL +(``https://x-access-token:TOKEN@github.com/...``), which persists in +``.git/config`` — acceptable only because those clones are ephemeral. The +per-repo store keeps its clone for a long time and the deep dive reads it, so a +token in ``.git/config`` would be a standing leak. + +Here the token is supplied to git transiently via ``GIT_ASKPASS`` (read from an +env var by a tiny helper script, never from argv), and the clone URL is plain. +Result: ``.git/config`` carries only ``https://host/owner/repo`` — no token, +not even the ``x-access-token`` username. "Practice what you preach." +""" + +from __future__ import annotations + +import asyncio +import os +import signal +import stat +import tempfile +from pathlib import Path + +from cliff.assessment.clone import ( + CloneError, + CloneTimeoutError, + redact_token, + validate_repo_url, +) + +__all__ = [ + "CloneError", + "CloneTimeoutError", + "askpass_response", + "clone_repo", + "refresh_repo", +] + +#: Env var the askpass helper reads the token from (never passed on argv). +TOKEN_ENV = "CLIFF_GIT_TOKEN" + +#: git calls the askpass program once per credential prompt, passing the prompt +#: text as ``$1``. We answer the username with the fixed GitHub-App token user +#: and the password with the token from the environment. +_ASKPASS_SCRIPT = ( + "#!/bin/sh\n" + 'case "$1" in\n' + ' *[Uu]sername*) printf "%s" "x-access-token" ;;\n' + f' *) printf "%s" "${TOKEN_ENV}" ;;\n' + "esac\n" +) + + +def askpass_response(prompt: str, token: str) -> str: + """Pure mirror of the askpass script's logic, for unit testing. + + A prompt mentioning "username" yields the fixed token-user; anything else + (the password prompt) yields the token. + """ + return "x-access-token" if "username" in prompt.lower() else token + + +def _clone_args(repo_url: str, target: Path, depth: int) -> list[str]: + """The git clone argv. The token is NEVER here — only in the env.""" + return [ + "git", + "-c", + "credential.helper=", # disable any helper that would cache the token + "clone", + "--depth", + str(depth), + "--single-branch", + "--", + repo_url, + str(target), + ] + + +def _git_env(token: str | None, askpass_path: Path | None) -> dict[str, str]: + env = {**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_NOSYSTEM": "1"} + if token and askpass_path is not None: + env["GIT_ASKPASS"] = str(askpass_path) + env[TOKEN_ENV] = token + return env + + +def _write_askpass() -> Path: + fd, path = tempfile.mkstemp(prefix="cliff-askpass-", suffix=".sh") + with os.fdopen(fd, "w") as f: + f.write(_ASKPASS_SCRIPT) + p = Path(path) + p.chmod(p.stat().st_mode | stat.S_IXUSR | stat.S_IRUSR) + return p + + +async def _run_git( + args: list[str], env: dict[str, str], *, timeout_s: float, token: str | None +) -> None: + proc = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + start_new_session=True, + ) + try: + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s) + except TimeoutError: + if proc.pid: + try: + os.killpg(proc.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + proc.kill() + await proc.wait() + raise CloneTimeoutError(f"git timed out after {timeout_s}s") from None + if proc.returncode != 0: + msg = redact_token(stderr.decode("utf-8", errors="replace"), token) + raise CloneError(f"git failed (exit {proc.returncode}): {msg.strip()}") + + +async def clone_repo( + repo_url: str, + *, + target: Path, + token: str | None, + timeout_s: float = 120.0, + depth: int = 1, +) -> None: + """Clone *repo_url* into *target* without persisting the token. + + When a token is supplied the host is validated against the GitHub allowlist + (we won't hand a token to an arbitrary host). The token reaches git only via + ``GIT_ASKPASS`` + env, so the resulting ``.git/config`` stays credential-less. + """ + has_token = bool(token) + if has_token: + validate_repo_url(repo_url, has_token=True) + askpass_path = _write_askpass() if has_token else None + try: + await _run_git( + _clone_args(repo_url, target, depth), + _git_env(token, askpass_path), + timeout_s=timeout_s, + token=token, + ) + finally: + if askpass_path is not None: + askpass_path.unlink(missing_ok=True) + + +async def refresh_repo( + target: Path, *, token: str | None, timeout_s: float = 120.0 +) -> None: + """Fetch the latest remote tip and hard-reset the working tree to it. + + Used to bring a cached clone up to date instead of re-cloning (ADR-0053 §4). + """ + has_token = bool(token) + askpass_path = _write_askpass() if has_token else None + env = _git_env(token, askpass_path) + try: + await _run_git( + ["git", "-C", str(target), "-c", "credential.helper=", "fetch", "origin"], + env, + timeout_s=timeout_s, + token=token, + ) + await _run_git( + ["git", "-C", str(target), "reset", "--hard", "FETCH_HEAD"], + env, + timeout_s=timeout_s, + token=token, + ) + finally: + if askpass_path is not None: + askpass_path.unlink(missing_ok=True) diff --git a/backend/cliff/repos/gc.py b/backend/cliff/repos/gc.py new file mode 100644 index 00000000..19cb168f --- /dev/null +++ b/backend/cliff/repos/gc.py @@ -0,0 +1,53 @@ +"""GC for the per-repo store's cached clones (ADR-0053 §4). + +The app has no disk-retention machinery. A long-lived clone per repo would grow +unbounded, so this evicts the *clones* (the big, re-clonable part) least-recently- +used until the total clone footprint is back under budget. The small JSON +profile artifacts — the actual value — are kept; only ``/repo/`` is removed, +so an evicted repo just re-clones on its next profile build. +""" + +from __future__ import annotations + +import shutil +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + +def _dir_size(path: Path) -> int: + return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + + +def gc_repo_clones(base_dir: Path, *, max_total_bytes: int) -> list[str]: + """Evict cached clones LRU until total clone size is within budget. + + Returns the list of repo ids whose clone was removed (most-stale first). + Profile artifacts are never touched. + """ + if not base_dir.exists(): + return [] + + clones: list[tuple[float, int, Path, str]] = [] + for repo_dir in base_dir.iterdir(): + clone = repo_dir / "repo" + if repo_dir.is_dir() and clone.is_dir(): + clones.append( + (clone.stat().st_mtime, _dir_size(clone), clone, repo_dir.name) + ) + + total = sum(size for _, size, _, _ in clones) + if total <= max_total_bytes: + return [] + + # Oldest (least recently used) first. + clones.sort(key=lambda c: c[0]) + evicted: list[str] = [] + for _mtime, size, clone, repo_id in clones: + if total <= max_total_bytes: + break + shutil.rmtree(clone) + evicted.append(repo_id) + total -= size + return evicted diff --git a/backend/tests/repos/test_clone.py b/backend/tests/repos/test_clone.py new file mode 100644 index 00000000..6a112149 --- /dev/null +++ b/backend/tests/repos/test_clone.py @@ -0,0 +1,162 @@ +"""Tests for the credential-less clone + GC (ADR-0053 §4). + +Network is avoided: the clone/refresh tests run against a local bare git repo. +The security property (no token in argv / in .git/config) is covered by unit +tests on the pure pieces plus the clean-config assertion after a real local clone. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from cliff.repos.clone import ( + TOKEN_ENV, + _clone_args, + _git_env, + askpass_response, + clone_repo, + refresh_repo, +) +from cliff.repos.gc import gc_repo_clones + +TOKEN = "ghp_supersecrettoken" + + +# ── the token never reaches argv (only the env) ───────────────────────────── + + +def test_clone_args_never_contain_the_token(): + args = _clone_args("https://github.com/acme/web", "/tmp/x", depth=1) + assert TOKEN not in " ".join(args) + assert "https://github.com/acme/web" in args # plain url, no creds + + +def test_token_lives_only_in_env(): + env = _git_env(TOKEN, askpass_path="/tmp/askpass.sh") + assert env[TOKEN_ENV] == TOKEN + assert env["GIT_ASKPASS"] == "/tmp/askpass.sh" + assert env["GIT_TERMINAL_PROMPT"] == "0" + + +def test_no_askpass_env_without_token(): + env = _git_env(None, askpass_path=None) + assert TOKEN_ENV not in env + assert "GIT_ASKPASS" not in env + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + ("Username for 'https://github.com': ", "x-access-token"), + ("Password for 'https://x-access-token@github.com': ", TOKEN), + ], +) +def test_askpass_response(prompt, expected): + assert askpass_response(prompt, TOKEN) == expected + + +# ── real clone/refresh against a local bare repo (offline) ────────────────── + + +def _git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + env={"GIT_CONFIG_NOSYSTEM": "1", "HOME": "/tmp", "PATH": "/usr/bin:/bin"}, + ) + + +@pytest.fixture +def bare_remote(tmp_path): + """A local bare repo (one commit) + a working clone that tracks it. + + Returns ``(work, bare)``: ``bare`` is the clone source for the tests; + ``work`` has ``origin=bare`` so a test can commit + push a new commit. + """ + seed = tmp_path / "seed" + seed.mkdir() + _git("init", "-q", "-b", "main", cwd=seed) + _git("config", "user.email", "t@t.t", cwd=seed) + _git("config", "user.name", "t", cwd=seed) + (seed / "hello.txt").write_text("v1\n") + _git("add", "hello.txt", cwd=seed) + _git("commit", "-q", "-m", "v1", cwd=seed) + bare = tmp_path / "remote.git" + _git("clone", "-q", "--bare", str(seed), str(bare)) + + work = tmp_path / "work" + _git("clone", "-q", f"file://{bare}", str(work)) + _git("config", "user.email", "t@t.t", cwd=work) + _git("config", "user.name", "t", cwd=work) + return work, bare + + +async def test_clone_repo_offline_no_token_clean_config(bare_remote, tmp_path): + _work, bare = bare_remote + target = tmp_path / "clone" + await clone_repo(f"file://{bare}", target=target, token=None, timeout_s=30) + assert (target / "hello.txt").read_text() == "v1\n" + # The security invariant: no token, not even x-access-token, in the config. + config = (target / ".git" / "config").read_text() + assert "x-access-token" not in config + assert TOKEN not in config + + +async def test_refresh_repo_pulls_new_commit(bare_remote, tmp_path): + work, bare = bare_remote + target = tmp_path / "clone" + await clone_repo(f"file://{bare}", target=target, token=None, timeout_s=30) + + # New commit lands on the remote. + (work / "hello.txt").write_text("v2\n") + _git("commit", "-q", "-am", "v2", cwd=work) + _git("push", "-q", "origin", "main", cwd=work) + + await refresh_repo(target, token=None, timeout_s=30) + assert (target / "hello.txt").read_text() == "v2\n" + + +# ── GC ────────────────────────────────────────────────────────────────────── + + +def _make_clone(base, repo_id, nbytes): + clone = base / repo_id / "repo" + clone.mkdir(parents=True) + (clone / "blob.bin").write_bytes(b"x" * nbytes) + # keep a profile artifact alongside to prove GC never removes it + (base / repo_id / "profile.json").write_text("{}") + return clone + + +def test_gc_under_budget_is_noop(tmp_path): + base = tmp_path / "repos" + _make_clone(base, "a", 100) + assert gc_repo_clones(base, max_total_bytes=10_000) == [] + assert (base / "a" / "repo").exists() + + +def test_gc_evicts_lru_clone_keeps_artifacts(tmp_path): + import os + import time + + base = tmp_path / "repos" + old = _make_clone(base, "old", 5_000) + _make_clone(base, "new", 5_000) + # Make "old" least-recently-used. + past = time.time() - 1000 + os.utime(old, (past, past)) + + evicted = gc_repo_clones(base, max_total_bytes=6_000) + assert evicted == ["old"] + assert not (base / "old" / "repo").exists() + assert (base / "new" / "repo").exists() + # GC never removes the profile artifacts — only the re-clonable clone. + assert (base / "old" / "profile.json").exists() + + +def test_gc_empty_base(tmp_path): + assert gc_repo_clones(tmp_path / "nope", max_total_bytes=10) == [] From 6848570931a472fe4f4d2a1e65d9130fb20728ed Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 22:26:21 +0300 Subject: [PATCH 04/34] =?UTF-8?q?feat(repos):=20ProfileRunner=20=E2=80=94?= =?UTF-8?q?=20per-repo=20profile=20build=20orchestration=20(ADR-0053=20P1.?= =?UTF-8?q?3/1.7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ties the mutex + clone sync + store + manifest + status lifecycle into one build flow. Builders / clone-sync / head-sha / token lookup are injected, so the whole backbone is proven end-to-end with fakes (mutex-skip, error→status, token-passthrough, artifacts+manifest+digest). The real LLM builders satisfy the ProfileBuilder shape. 4 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/repos/profile_runner.py | 104 ++++++++++++++++ backend/tests/repos/test_profile_runner.py | 131 +++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 backend/cliff/repos/profile_runner.py create mode 100644 backend/tests/repos/test_profile_runner.py diff --git a/backend/cliff/repos/profile_runner.py b/backend/cliff/repos/profile_runner.py new file mode 100644 index 00000000..30dd3a34 --- /dev/null +++ b/backend/cliff/repos/profile_runner.py @@ -0,0 +1,104 @@ +"""ProfileRunner — orchestrates one per-repo Project-profile build (ADR-0053). + +Ties together the pieces built in Phase 1: get-or-create the repo, acquire the +one-build-per-repo mutex, sync the cached clone, run the profile builders, write +their artifacts + the manifest + the generated digest, and release the mutex +with a terminal status. + +The builders, the clone sync, the HEAD-sha read, and the token lookup are all +*injected* — so the whole flow is testable end-to-end with fakes, and the real +LLM builders (repo_profiler/code_map/threat_history) just satisfy the +``ProfileBuilder`` shape. Eager-at-scan wiring (PRD-0009) calls ``build`` as a +background task. +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from cliff.repos.dao import ( + finish_profile, + get_or_create_repo, + get_repo, + try_begin_profile, +) + +if TYPE_CHECKING: + from pathlib import Path + + import aiosqlite + + from cliff.models.repo import Repo + from cliff.repos.repo_dir_manager import RepoDirManager + +logger = logging.getLogger(__name__) + +#: A builder reads the cached clone and returns one artifact dict. +ProfileBuilder = Callable[["Path"], Awaitable[dict]] +#: Bring the cached clone to the current remote tip (clone or fetch+reset). +SyncClone = Callable[[str, "Path", "str | None"], Awaitable[None]] +#: Read the checked-out HEAD sha of a clone. +HeadSha = Callable[["Path"], Awaitable[str]] +#: Look up the GitHub token (from the vault), or None when unconfigured. +TokenProvider = Callable[[], Awaitable["str | None"]] + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +class ProfileRunner: + def __init__( + self, + db: aiosqlite.Connection, + dir_mgr: RepoDirManager, + *, + builders: dict[str, ProfileBuilder], + sync_clone: SyncClone, + head_sha: HeadSha, + token_provider: TokenProvider, + ) -> None: + self._db = db + self._dir = dir_mgr + self._builders = builders + self._sync_clone = sync_clone + self._head_sha = head_sha + self._token_provider = token_provider + + async def build(self, repo_url: str) -> Repo | None: + """Build (or rebuild) the Project profile for *repo_url*. + + Returns the repo row. If a build is already in progress the mutex is not + acquired and the existing row is returned untouched (no double work). + """ + repo = await get_or_create_repo(self._db, repo_url) + if not await try_begin_profile(self._db, repo.id): + logger.info("profile build already in progress for %s", repo.canonical_url) + return await get_repo(self._db, repo.id) + + try: + token = await self._token_provider() + clone_dir = self._dir.clone_dir(repo.id) + await self._sync_clone(repo.canonical_url, clone_dir, token) + sha = await self._head_sha(clone_dir) + + for name, builder in self._builders.items(): + self._dir.write_artifact(repo.id, name, await builder(clone_dir)) + + self._dir.write_manifest(repo.id, source_sha=sha, built_at=_now()) + self._dir.regenerate_profile_md(repo.id) + + return await finish_profile( + self._db, + repo.id, + status="ready", + sha=sha, + profile_dir=self._dir.repo_dir(repo.id), + ) + except Exception: + logger.exception("profile build failed for %s", repo.canonical_url) + await finish_profile(self._db, repo.id, status="error") + raise diff --git a/backend/tests/repos/test_profile_runner.py b/backend/tests/repos/test_profile_runner.py new file mode 100644 index 00000000..553f997c --- /dev/null +++ b/backend/tests/repos/test_profile_runner.py @@ -0,0 +1,131 @@ +"""End-to-end test of the per-repo profile build flow with injected fakes. + +Proves the Phase-1 backbone (mutex + clone + store + manifest + status +lifecycle) without any real LLM or network — the real builders just satisfy the +injected ``ProfileBuilder`` shape. +""" + +from __future__ import annotations + +import pytest + +from cliff.repos.dao import get_repo, try_begin_profile +from cliff.repos.profile_runner import ProfileRunner +from cliff.repos.repo_dir_manager import RepoDirManager + +URL = "https://github.com/acme/web" + + +class _Fakes: + def __init__(self): + self.builder_calls: list[str] = [] + self.clone_calls: list[tuple[str, str | None]] = [] + + def builder(self, name, payload): + async def _b(clone_dir): + self.builder_calls.append(name) + return payload + + return _b + + async def sync_clone(self, canonical_url, clone_dir, token): + self.clone_calls.append((canonical_url, token)) + clone_dir.mkdir(parents=True, exist_ok=True) + + async def head_sha(self, clone_dir): + return "abcd1234" + + async def token(self): + return "ghp_tok" + + +@pytest.fixture +def runner(db, tmp_path): + fakes = _Fakes() + mgr = RepoDirManager(tmp_path / "repos") + r = ProfileRunner( + db, + mgr, + builders={ + "profile": fakes.builder("profile", {"kind": "service"}), + "code_map": fakes.builder("code_map", {"ships_roots": ["src/**"]}), + "threat": fakes.builder("threat", {"prior_issues": []}), + }, + sync_clone=fakes.sync_clone, + head_sha=fakes.head_sha, + token_provider=fakes.token, + ) + return r, fakes, mgr + + +async def test_build_happy_path(runner, db): + r, fakes, mgr = runner + repo = await r.build(URL) + + assert repo.profile_status == "ready" + assert repo.last_profiled_sha == "abcd1234" + assert repo.profile_dir == mgr.repo_dir(repo.id) + assert repo.profiled_at is not None + # All three artifacts written + manifest + digest. + assert mgr.read_artifact(repo.id, "profile") == {"kind": "service"} + assert mgr.read_artifact(repo.id, "code_map") == {"ships_roots": ["src/**"]} + assert set(mgr.read_manifest(repo.id)["artifacts"]) == {"profile", "code_map", "threat"} + assert mgr.read_manifest(repo.id)["source_sha"] == "abcd1234" + assert mgr.profile_md_path(repo.id).exists() + assert sorted(fakes.builder_calls) == ["code_map", "profile", "threat"] + + +async def test_token_is_passed_to_clone(runner): + r, fakes, _mgr = runner + await r.build(URL) + assert fakes.clone_calls == [(URL, "ghp_tok")] + + +async def test_build_skips_when_mutex_already_held(runner, db): + r, fakes, _mgr = runner + # Simulate a build already in flight by grabbing the mutex first. + from cliff.repos.dao import get_or_create_repo + + repo = await get_or_create_repo(db, URL) + assert await try_begin_profile(db, repo.id) is True + + result = await r.build(URL) + # The runner must not run builders while another build holds the mutex. + assert result.profile_status == "building" + assert fakes.builder_calls == [] + + +async def test_builder_error_marks_error_and_reraises(db, tmp_path): + mgr = RepoDirManager(tmp_path / "repos") + + async def boom(clone_dir): + raise RuntimeError("profiler crashed") + + async def sync_clone(url, clone_dir, token): + clone_dir.mkdir(parents=True, exist_ok=True) + + async def head_sha(clone_dir): + return "sha" + + async def token(): + return None + + r = ProfileRunner( + db, + mgr, + builders={"profile": boom}, + sync_clone=sync_clone, + head_sha=head_sha, + token_provider=token, + ) + with pytest.raises(RuntimeError, match="profiler crashed"): + await r.build(URL) + + repo = await get_repo(db, (await get_repo_id(db))) + assert repo.profile_status == "error" + + +async def get_repo_id(db): + from cliff.repos.dao import list_repos + + return (await list_repos(db))[0].id From a882b21f801bd370642e506b0dcde28ca40c51d5 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 22:29:26 +0300 Subject: [PATCH 05/34] feat(repos): the three Project-profile builder agents (ADR-0053 P1.6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repo_profiler / code_map / threat_history — read-only in-process Pydantic AI agents emitting typed artifacts (RepoProfile/CodeMap/ThreatHistory, extra=allow). They satisfy the ProfileBuilder shape so ProfileRunner drives them, reuse the existing read tool (workspace_dir → clone), and are read-only by design. Driven end-to-end with TestModel; quality is the key-gated eval. 6 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/repos/profile_agents.py | 122 +++++++++++++++++++++ backend/cliff/repos/schemas.py | 85 ++++++++++++++ backend/tests/repos/test_profile_agents.py | 75 +++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 backend/cliff/repos/profile_agents.py create mode 100644 backend/cliff/repos/schemas.py create mode 100644 backend/tests/repos/test_profile_agents.py diff --git a/backend/cliff/repos/profile_agents.py b/backend/cliff/repos/profile_agents.py new file mode 100644 index 00000000..ea12b6ca --- /dev/null +++ b/backend/cliff/repos/profile_agents.py @@ -0,0 +1,122 @@ +"""The three Project-profile builder agents (ADR-0053 §3, Phase 1.6). + +Each is an in-process Pydantic AI agent (ADR-0047) that reads the cached clone +(read-only) and emits one typed artifact. They satisfy the ``ProfileBuilder`` +shape (``async (clone_dir) -> dict``) so :class:`ProfileRunner` can drive them. + +Deps reuse: profile builders run on a *clone*, not a finding workspace, but the +``read`` tool is keyed on ``WorkspaceDeps.workspace_dir`` — so we reuse +``WorkspaceDeps`` with ``workspace_dir`` pointing at the clone and an empty +``finding``. That reuses the entire tool + runtime path with zero duplication +("delete before adding"); the repo metadata is threaded through the task prompt. + +Read-only by design (the whole profiling tier touches nothing): the only tool is +``read``. ``PROFILE_BUILDER_TOOLS`` makes that boundary one assertable thing. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic_ai import Agent +from pydantic_ai.usage import UsageLimits + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools.read import read +from cliff.repos.schemas import CodeMap, RepoProfile, ThreatHistory + +if TYPE_CHECKING: + from pathlib import Path + + from pydantic_ai.models import Model + + from cliff.repos.profile_runner import ProfileBuilder + +#: Read-only — the profiling tier never edits, runs, or pushes anything. +PROFILE_BUILDER_TOOLS = (read,) + +#: A profile build reads a handful of files (README, manifests, a tree sample); +#: cap requests so a weak model can't loop on the read tool. +PROFILE_REQUEST_LIMIT = 15 + +_PROFILER_PROMPT = """\ +Profile this repository. Use the `read` tool to open its README, package \ +manifest(s) (package.json, pyproject.toml, go.mod, Cargo.toml, etc.), any \ +Dockerfile, and the obvious entry points. Determine: what kind of project it is \ +(library / service / cli / self_hosted_app / monolith), how it is deployed and \ +run, whether it is internet-facing, its main attack-surface entry points (with \ +file:line where you can), and how to build and run it. Write a one-paragraph \ +plain-English summary. Read sparingly — you do not need every file.""" + +_CODE_MAP_PROMPT = """\ +Map which of this repository's code actually ships in production versus what \ +does not. Use the `read` tool to inspect the directory layout and a sample of \ +files. Classify path globs into: ships, test, fixture, example, docs, build, \ +vendored, dead — each with a short reason. List the top-level ships_roots and \ +excluded_roots. This is used to drop findings whose root cause lives only in \ +non-shipping code, so be accurate about what is test/fixture/vendored.""" + +_THREAT_PROMPT = """\ +Review this repository's security history. From any CHANGELOG, SECURITY.md, \ +advisory references, or security-relevant commit messages you can `read`, list \ +prior CVEs/GHSAs with their root-cause family and whether the fix addressed the \ +instance or the whole class. Note the weak-spot families that recur and the \ +areas of the codebase that have been historically fertile for bugs. If you find \ +no history, return empty lists — do not invent issues.""" + + +def _build_agent(model: Model, output_type: type) -> Agent: + return Agent( + model=model, + output_type=output_type, + deps_type=WorkspaceDeps, + tools=list(PROFILE_BUILDER_TOOLS), + ) + + +def _make_builder(model: Model, output_type: type, prompt: str) -> ProfileBuilder: + agent = _build_agent(model, output_type) + + async def _build(clone_dir: Path) -> dict: + deps = WorkspaceDeps( + workspace_id="repo-profile", + workspace_dir=str(clone_dir), + finding={}, + ) + result = await agent.run( + prompt, deps=deps, usage_limits=UsageLimits(request_limit=PROFILE_REQUEST_LIMIT) + ) + return result.output.model_dump() + + return _build + + +def make_repo_profiler(model: Model) -> ProfileBuilder: + return _make_builder(model, RepoProfile, _PROFILER_PROMPT) + + +def make_code_map(model: Model) -> ProfileBuilder: + return _make_builder(model, CodeMap, _CODE_MAP_PROMPT) + + +def make_threat_history(model: Model) -> ProfileBuilder: + return _make_builder(model, ThreatHistory, _THREAT_PROMPT) + + +def make_profile_builders(model: Model) -> dict[str, ProfileBuilder]: + """The full builder set keyed by artifact name, ready for ProfileRunner.""" + return { + "profile": make_repo_profiler(model), + "code_map": make_code_map(model), + "threat": make_threat_history(model), + } + + +__all__ = [ + "PROFILE_BUILDER_TOOLS", + "PROFILE_REQUEST_LIMIT", + "make_code_map", + "make_profile_builders", + "make_repo_profiler", + "make_threat_history", +] diff --git a/backend/cliff/repos/schemas.py b/backend/cliff/repos/schemas.py new file mode 100644 index 00000000..8fc4587a --- /dev/null +++ b/backend/cliff/repos/schemas.py @@ -0,0 +1,85 @@ +"""Project-profile artifact schemas (ADR-0053 §3). + +The typed shapes the profile builders emit and the triage Deep dive (ADR-0052) +consumes. ``extra="allow"`` keeps them forward-compatible (a newer builder can +add a field without breaking an older reader); a breaking change bumps +``RepoDirManager.SCHEMA_VERSION`` and the artifact is rebuilt, not migrated. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel + +# ── Profile the project ────────────────────────────────────────────────────── + +RepoKind = Literal["library", "service", "cli", "self_hosted_app", "monolith", "unknown"] + + +class EntryPoint(BaseModel): + model_config = {"extra": "allow"} + kind: str # http | cli | queue | deserializer | webhook | ... + location: str | None = None # file:line + description: str | None = None + + +class RepoProfile(BaseModel): + """``profile.json`` — what the project is and how it runs.""" + + model_config = {"extra": "allow"} + kind: RepoKind = "unknown" + deployment: str | None = None + internet_facing: bool | None = None # tri-state: None = couldn't determine + entry_points: list[EntryPoint] = [] + trust_boundary: str | None = None + build_cmd: str | None = None + run_cmd: str | None = None + docker_present: bool = False + dockerfile_path: str | None = None + summary: str | None = None # one plain-English paragraph for PROFILE.md + + +# ── Map the live code ──────────────────────────────────────────────────────── + +PathCategory = Literal[ + "ships", "test", "fixture", "example", "docs", "build", "vendored", "dead" +] + + +class PathClass(BaseModel): + model_config = {"extra": "allow"} + glob: str + category: PathCategory + reason: str | None = None + + +class CodeMap(BaseModel): + """``code_map.json`` — what ships vs what doesn't.""" + + model_config = {"extra": "allow"} + ships_roots: list[str] = [] + excluded_roots: list[str] = [] + classified: list[PathClass] = [] + + +# ── Review past issues ─────────────────────────────────────────────────────── + +FixCoverage = Literal["instance", "class", "none"] + + +class PriorIssue(BaseModel): + model_config = {"extra": "allow"} + id: str # CVE / GHSA id + root_cause_family: str | None = None + fixed: FixCoverage | None = None + summary: str | None = None + + +class ThreatHistory(BaseModel): + """``threat.json`` — the repo's prior vulnerabilities and recurring weak spots.""" + + model_config = {"extra": "allow"} + prior_issues: list[PriorIssue] = [] + recurring_families: list[str] = [] + fertile_areas: list[str] = [] diff --git a/backend/tests/repos/test_profile_agents.py b/backend/tests/repos/test_profile_agents.py new file mode 100644 index 00000000..91f066c0 --- /dev/null +++ b/backend/tests/repos/test_profile_agents.py @@ -0,0 +1,75 @@ +"""Profile-builder agents — shape + read-only boundary + end-to-end with TestModel. + +Keyless: drives the agents with TestModel (no real LLM). Verdict quality is the +key-gated eval, later. These assert the builders are read-only, satisfy the +ProfileBuilder shape, and return validated artifact dicts. +""" + +from __future__ import annotations + +import pytest +from pydantic_ai.models.test import TestModel + +from cliff.agents.runtime.tools import bash, edit, gh, read, webfetch +from cliff.repos.profile_agents import ( + PROFILE_BUILDER_TOOLS, + make_code_map, + make_profile_builders, + make_repo_profiler, + make_threat_history, +) +from cliff.repos.schemas import CodeMap, RepoProfile, ThreatHistory + + +def test_profile_builders_are_read_only(): + """The whole profiling tier touches nothing — the only tool is read.""" + assert PROFILE_BUILDER_TOOLS == (read,) + for forbidden in (bash, edit, gh, webfetch): + assert forbidden not in PROFILE_BUILDER_TOOLS + + +@pytest.fixture +def clone(tmp_path): + d = tmp_path / "repo" + d.mkdir() + (d / "README.md").write_text("# Acme web\nA self-hosted web service.\n") + (d / "pyproject.toml").write_text("[project]\nname='acme'\n") + return d + + +async def test_repo_profiler_returns_valid_profile(clone): + builder = make_repo_profiler(TestModel(custom_output_args={"kind": "service"})) + out = await builder(clone) + # Round-trips through the schema (defaults filled, extra allowed). + parsed = RepoProfile.model_validate(out) + assert parsed.kind == "service" + + +async def test_code_map_returns_valid_map(clone): + builder = make_code_map( + TestModel(custom_output_args={"ships_roots": ["src/**"], "excluded_roots": ["tests/**"]}) + ) + out = await builder(clone) + parsed = CodeMap.model_validate(out) + assert parsed.ships_roots == ["src/**"] + assert parsed.excluded_roots == ["tests/**"] + + +async def test_threat_history_returns_valid_history(clone): + builder = make_threat_history(TestModel(custom_output_args={"recurring_families": ["ssti"]})) + out = await builder(clone) + parsed = ThreatHistory.model_validate(out) + assert parsed.recurring_families == ["ssti"] + + +async def test_threat_history_defaults_to_empty(clone): + builder = make_threat_history(TestModel(custom_output_args={})) + out = await builder(clone) + parsed = ThreatHistory.model_validate(out) + assert parsed.prior_issues == [] + + +def test_make_profile_builders_has_all_three(): + builders = make_profile_builders(TestModel()) + assert set(builders) == {"profile", "code_map", "threat"} + assert all(callable(b) for b in builders.values()) From 0a33b78309f8edd9f0928cd6b55c106ca559682d Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 22:58:05 +0300 Subject: [PATCH 06/34] feat(repos): eager profile build at scan time (ADR-0053/PRD-0009 P1.7) git_ops (real sync_clone clone-or-refresh + git_head_sha, offline-tested) + service.build_profile_runner / schedule_profile_build (assembles a real ProfileRunner from the canonical AI state + vault token, fires a tracked background task; best-effort no-op when no AI provider). Hooked into the assessment run route so a scan grounds triage. 16 tests; full offline e2e through the real git adapters. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/api/routes/assessment.py | 6 ++ backend/cliff/repos/git_ops.py | 54 +++++++++++++ backend/cliff/repos/service.py | 91 ++++++++++++++++++++++ backend/tests/repos/conftest.py | 34 ++++++++ backend/tests/repos/test_git_ops.py | 50 ++++++++++++ backend/tests/repos/test_profile_agents.py | 2 +- backend/tests/repos/test_service.py | 75 ++++++++++++++++++ 7 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 backend/cliff/repos/git_ops.py create mode 100644 backend/cliff/repos/service.py create mode 100644 backend/tests/repos/test_git_ops.py create mode 100644 backend/tests/repos/test_service.py diff --git a/backend/cliff/api/routes/assessment.py b/backend/cliff/api/routes/assessment.py index 10fa1667..89254383 100644 --- a/backend/cliff/api/routes/assessment.py +++ b/backend/cliff/api/routes/assessment.py @@ -262,6 +262,12 @@ async def run_assessment( schedule_assessment_run( http_request.app, db, engine, assessment.id, request.repo_url ) + # Eager Project-profile build (ADR-0053 / PRD-0009): kicks off alongside the + # scan so triage is grounded later. Best-effort — never blocks the scan, and + # no-ops when no AI provider is configured. + from cliff.repos.service import schedule_profile_build + + schedule_profile_build(http_request.app, db, request.repo_url) return AssessmentRunResponse(assessment_id=assessment.id, status="pending") diff --git a/backend/cliff/repos/git_ops.py b/backend/cliff/repos/git_ops.py new file mode 100644 index 00000000..6ddcf8bd --- /dev/null +++ b/backend/cliff/repos/git_ops.py @@ -0,0 +1,54 @@ +"""Real git adapters for the per-repo clone (ADR-0053 Phase 1.7). + +The concrete ``sync_clone`` / ``head_sha`` the ProfileRunner is injected with in +production. Both are thin wrappers over :mod:`cliff.repos.clone`; kept separate +so the runner stays testable with fakes and these stay offline-testable against a +local repo. +""" + +from __future__ import annotations + +import asyncio +import shutil +from typing import TYPE_CHECKING + +from cliff.repos.clone import clone_repo, refresh_repo + +if TYPE_CHECKING: + from pathlib import Path + + +async def git_head_sha(clone_dir: Path) -> str: + """Return the checked-out HEAD sha of *clone_dir* (full 40-char).""" + proc = await asyncio.create_subprocess_exec( + "git", + "-C", + str(clone_dir), + "rev-parse", + "HEAD", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await proc.communicate() + if proc.returncode != 0: + raise RuntimeError( + f"git rev-parse failed in {clone_dir}: {err.decode('utf-8', 'replace').strip()}" + ) + return out.decode("utf-8").strip() + + +async def sync_clone( + canonical_url: str, clone_dir: Path, token: str | None, *, timeout_s: float = 120.0 +) -> None: + """Bring *clone_dir* to the remote tip: refresh if it's a clone, else clone. + + A stale/partial directory (exists but isn't a git repo) is removed and + re-cloned, so a half-finished previous build can't wedge the next one. + """ + if (clone_dir / ".git").exists(): + await refresh_repo(clone_dir, token=token, timeout_s=timeout_s) + return + if clone_dir.exists(): + shutil.rmtree(clone_dir) + clone_dir.parent.mkdir(parents=True, exist_ok=True) + await clone_repo(canonical_url, target=clone_dir, token=token, timeout_s=timeout_s) diff --git a/backend/cliff/repos/service.py b/backend/cliff/repos/service.py new file mode 100644 index 00000000..8cb0ec1d --- /dev/null +++ b/backend/cliff/repos/service.py @@ -0,0 +1,91 @@ +"""Eager profile-build wiring (ADR-0053 / PRD-0009 Phase 1.7). + +Assembles a real :class:`ProfileRunner` from the canonical AI state + the vault +GitHub token, and schedules a build as a tracked background task at scan time — +mirroring ``schedule_assessment_run``. Best-effort: if no AI provider is +configured the build is skipped (never blocks or breaks the assessment). +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING + +from cliff.config import settings +from cliff.repos.git_ops import git_head_sha, sync_clone +from cliff.repos.profile_agents import make_profile_builders +from cliff.repos.profile_runner import ProfileRunner, TokenProvider +from cliff.repos.repo_dir_manager import RepoDirManager + +if TYPE_CHECKING: + import aiosqlite + from fastapi import FastAPI + from pydantic_ai.models import Model + +logger = logging.getLogger(__name__) + + +def default_repo_dir_manager() -> RepoDirManager: + return RepoDirManager(settings.resolve_data_dir() / "repos") + + +def build_profile_runner( + db: aiosqlite.Connection, + *, + model: Model, + token_provider: TokenProvider, + dir_mgr: RepoDirManager | None = None, +) -> ProfileRunner: + """Assemble a production ProfileRunner (real clone + sha; injected model/token).""" + return ProfileRunner( + db, + dir_mgr or default_repo_dir_manager(), + builders=make_profile_builders(model), + sync_clone=sync_clone, + head_sha=git_head_sha, + token_provider=token_provider, + ) + + +def schedule_profile_build( + app: FastAPI, db: aiosqlite.Connection, repo_url: str +) -> asyncio.Task[None] | None: + """Fire-and-track an eager Project-profile build for *repo_url*. + + Returns the task, or ``None`` when skipped (no AI provider configured / the + model can't be built). Always best-effort — never raises into the caller. + """ + env = dict(getattr(app.state, "ai_env_cache", {}) or {}) + model_id = getattr(app.state, "ai_model_cache", None) + if not model_id or not env: + logger.info("profile build skipped for %s — no AI provider configured", repo_url) + return None + + try: + from cliff.agents.runtime.provider import build_model + + model = build_model(env, model_id) + except Exception: + logger.warning("profile build skipped for %s — model unavailable", repo_url, exc_info=True) + return None + + from cliff.api._engine_dep import _github_token_from_integration + + runner = build_profile_runner( + db, model=model, token_provider=_github_token_from_integration + ) + + tasks: set[asyncio.Task[None]] = getattr(app.state, "profile_tasks", None) or set() + + async def _run() -> None: + try: + await runner.build(repo_url) + except Exception: + logger.exception("eager profile build failed for %s", repo_url) + + task = asyncio.create_task(_run(), name=f"profile:{repo_url}") + tasks.add(task) + task.add_done_callback(tasks.discard) + app.state.profile_tasks = tasks + return task diff --git a/backend/tests/repos/conftest.py b/backend/tests/repos/conftest.py index b814460b..4d85788a 100644 --- a/backend/tests/repos/conftest.py +++ b/backend/tests/repos/conftest.py @@ -2,6 +2,8 @@ from __future__ import annotations +import subprocess + import pytest @@ -15,3 +17,35 @@ async def db(): yield conn finally: await close_db() + + +@pytest.fixture +def git_run(): + """Run a git command offline (no global config / credential prompts).""" + + def _run(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + env={"GIT_CONFIG_NOSYSTEM": "1", "HOME": "/tmp", "PATH": "/usr/bin:/bin"}, + ) + + return _run + + +@pytest.fixture +def bare_repo(tmp_path, git_run): + """A local bare repo (one commit) usable as an offline clone source.""" + seed = tmp_path / "seed" + seed.mkdir() + git_run("init", "-q", "-b", "main", cwd=seed) + git_run("config", "user.email", "t@t.t", cwd=seed) + git_run("config", "user.name", "t", cwd=seed) + (seed / "README.md").write_text("# svc\nv1\n") + git_run("add", "README.md", cwd=seed) + git_run("commit", "-q", "-m", "init", cwd=seed) + bare = tmp_path / "remote.git" + git_run("clone", "-q", "--bare", str(seed), str(bare)) + return bare diff --git a/backend/tests/repos/test_git_ops.py b/backend/tests/repos/test_git_ops.py new file mode 100644 index 00000000..17d1250b --- /dev/null +++ b/backend/tests/repos/test_git_ops.py @@ -0,0 +1,50 @@ +"""Offline tests for the real git adapters (ADR-0053 Phase 1.7).""" + +from __future__ import annotations + +import pytest + +from cliff.repos.git_ops import git_head_sha, sync_clone + + +async def test_sync_clone_fresh_then_head_sha(bare_repo, tmp_path): + clone = tmp_path / "store" / "rid" / "repo" + await sync_clone(f"file://{bare_repo}", clone, None, timeout_s=30) + assert (clone / "README.md").read_text().startswith("# svc") + sha = await git_head_sha(clone) + assert len(sha) == 40 + + +async def test_sync_clone_refreshes_existing(bare_repo, tmp_path, git_run): + clone = tmp_path / "store" / "rid" / "repo" + await sync_clone(f"file://{bare_repo}", clone, None, timeout_s=30) + first = await git_head_sha(clone) + + # Push a new commit to the remote via a throwaway work clone. + work = tmp_path / "work" + git_run("clone", "-q", f"file://{bare_repo}", str(work)) + git_run("config", "user.email", "t@t.t", cwd=work) + git_run("config", "user.name", "t", cwd=work) + (work / "README.md").write_text("# svc\nv2\n") + git_run("commit", "-q", "-am", "v2", cwd=work) + git_run("push", "-q", "origin", "main", cwd=work) + + # Second sync takes the refresh path (.git exists) and updates the tree. + await sync_clone(f"file://{bare_repo}", clone, None, timeout_s=30) + assert (clone / "README.md").read_text() == "# svc\nv2\n" + assert await git_head_sha(clone) != first + + +async def test_sync_clone_replaces_stale_non_git_dir(bare_repo, tmp_path): + clone = tmp_path / "store" / "rid" / "repo" + clone.mkdir(parents=True) + (clone / "leftover.txt").write_text("junk from a half-finished build") + # No .git → the dir is removed and re-cloned cleanly. + await sync_clone(f"file://{bare_repo}", clone, None, timeout_s=30) + assert (clone / "README.md").exists() + assert not (clone / "leftover.txt").exists() + + +async def test_git_head_sha_raises_on_non_repo(tmp_path): + with pytest.raises(RuntimeError, match="rev-parse failed"): + await git_head_sha(tmp_path) diff --git a/backend/tests/repos/test_profile_agents.py b/backend/tests/repos/test_profile_agents.py index 91f066c0..db2ac947 100644 --- a/backend/tests/repos/test_profile_agents.py +++ b/backend/tests/repos/test_profile_agents.py @@ -23,7 +23,7 @@ def test_profile_builders_are_read_only(): """The whole profiling tier touches nothing — the only tool is read.""" - assert PROFILE_BUILDER_TOOLS == (read,) + assert (read,) == PROFILE_BUILDER_TOOLS for forbidden in (bash, edit, gh, webfetch): assert forbidden not in PROFILE_BUILDER_TOOLS diff --git a/backend/tests/repos/test_service.py b/backend/tests/repos/test_service.py new file mode 100644 index 00000000..a77ca1d1 --- /dev/null +++ b/backend/tests/repos/test_service.py @@ -0,0 +1,75 @@ +"""Tests for the eager profile-build wiring (ADR-0053 Phase 1.7).""" + +from __future__ import annotations + +from types import SimpleNamespace + +from pydantic_ai.models.test import TestModel + +from cliff.repos import git_ops +from cliff.repos.profile_agents import make_profile_builders +from cliff.repos.profile_runner import ProfileRunner +from cliff.repos.repo_dir_manager import RepoDirManager +from cliff.repos.service import build_profile_runner, schedule_profile_build + + +def test_build_profile_runner_wires_production_adapters(db, tmp_path): + """The production runner uses the real git adapters + all three builders.""" + mgr = RepoDirManager(tmp_path / "repos") + + async def token(): + return None + + runner = build_profile_runner( + db, model=TestModel(), token_provider=token, dir_mgr=mgr + ) + assert runner._sync_clone is git_ops.sync_clone + assert runner._head_sha is git_ops.git_head_sha + assert set(runner._builders) == {"profile", "code_map", "threat"} + + +async def test_end_to_end_with_real_git_adapters(db, bare_repo, tmp_path): + """Real git_ops.sync_clone + real git_head_sha + TestModel builders profile a + canonical (github-shaped) repo end to end, offline — the clone source is the + local bare repo, everything else is production code.""" + mgr = RepoDirManager(tmp_path / "repos") + + async def redirect_sync(canonical_url, clone_dir, token): + # Exercise the REAL adapter, just pointed at the offline bare repo. + await git_ops.sync_clone(f"file://{bare_repo}", clone_dir, None, timeout_s=30) + + async def token(): + return None + + runner = ProfileRunner( + db, + mgr, + builders=make_profile_builders(TestModel(custom_output_args={"kind": "service"})), + sync_clone=redirect_sync, + head_sha=git_ops.git_head_sha, + token_provider=token, + ) + repo = await runner.build("https://github.com/acme/web") + + assert repo.canonical_url == "https://github.com/acme/web" + assert repo.profile_status == "ready" + assert len(repo.last_profiled_sha) == 40 # a real sha read from the real clone + assert mgr.read_artifact(repo.id, "profile")["kind"] == "service" + assert (mgr.clone_dir(repo.id) / "README.md").exists() + + +async def test_schedule_profile_build_skips_without_ai(db): + """No configured AI provider → best-effort no-op (never blocks the scan).""" + app = SimpleNamespace(state=SimpleNamespace(ai_env_cache={}, ai_model_cache=None)) + assert schedule_profile_build(app, db, "https://github.com/a/b") is None + + +async def test_schedule_profile_build_skips_on_bad_model(db): + """A configured-but-unbuildable model is skipped, not raised.""" + app = SimpleNamespace( + state=SimpleNamespace( + ai_env_cache={"ANTHROPIC_API_KEY": "x"}, + ai_model_cache="not-a-valid-id-without-slash", + ) + ) + assert schedule_profile_build(app, db, "https://github.com/a/b") is None From 5bd47d4a1d061704242983598ac83d512f2109ba Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:01:18 +0300 Subject: [PATCH 07/34] feat(api): repo Project-profile endpoints (ADR-0053/PRD-0009 P1.8 backend) GET /api/repos/profile (status + freshness + generated PROFILE.md, resolving the current repo from the GitHub integration when not given) and POST /api/repos/profile/rebuild (schedules the eager build; skipped cleanly without an AI provider). Registered the router; OpenAPI snapshot regenerated. 5 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/api/routes/repos.py | 109 +++++++++++++ backend/cliff/main.py | 2 + backend/tests/api/openapi_snapshot.json | 200 ++++++++++++++++++++++++ backend/tests/test_routes_repos.py | 59 +++++++ 4 files changed, 370 insertions(+) create mode 100644 backend/cliff/api/routes/repos.py create mode 100644 backend/tests/test_routes_repos.py diff --git a/backend/cliff/api/routes/repos.py b/backend/cliff/api/routes/repos.py new file mode 100644 index 00000000..1f162ab3 --- /dev/null +++ b/backend/cliff/api/routes/repos.py @@ -0,0 +1,109 @@ +"""Repo Project-profile API (ADR-0053 / PRD-0009 Phase 1.8). + +Surfaces the per-repo profile freshness for the dashboard ("Cliff understands +your project; built N ago") and a re-profile action. Read path is the ``repo`` +row + the generated ``PROFILE.md``; the rebuild path schedules the same eager +build the scan uses. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TCH003 — Pydantic needs this at runtime +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException +from fastapi import Request as FastAPIRequest +from pydantic import BaseModel + +from cliff.db.connection import get_db +from cliff.repos.dao import get_repo_by_url +from cliff.repos.service import schedule_profile_build + +router = APIRouter(prefix="/repos", tags=["repos"]) + + +class RepoProfileStatus(BaseModel): + repo_url: str | None = None + #: none | building | ready | stale | error + status: str = "none" + profiled_at: datetime | None = None + last_profiled_sha: str | None = None + #: The generated PROFILE.md digest, when a profile exists. + profile_md: str | None = None + + +class RebuildBody(BaseModel): + repo_url: str | None = None + + +class RebuildResponse(BaseModel): + #: scheduled | skipped + status: str + repo_url: str | None = None + #: present when skipped (e.g. ``no_ai_provider``) + reason: str | None = None + + +async def _resolve_repo_url(db, explicit: str | None) -> str | None: + """The explicit url, else the connected GitHub integration's repo.""" + if explicit and explicit.strip(): + return explicit.strip() + from cliff.db.repo_integration import list_integrations + + integrations = await list_integrations(db) + github = next( + (i for i in integrations if i.provider_name.lower() == "github" and i.enabled), + None, + ) + if github and github.config: + url = github.config.get("repo_url") + if url: + return url + return None + + +@router.get("/profile", response_model=RepoProfileStatus) +async def get_profile( + repo_url: str | None = None, db=Depends(get_db) +) -> RepoProfileStatus: + """The Project-profile status + freshness for the (current or given) repo.""" + url = await _resolve_repo_url(db, repo_url) + if url is None: + return RepoProfileStatus() + + repo = await get_repo_by_url(db, url) + if repo is None: + return RepoProfileStatus(repo_url=url, status="none") + + profile_md: str | None = None + if repo.profile_dir: + md_path = Path(repo.profile_dir) / "PROFILE.md" + if md_path.exists(): + try: + profile_md = md_path.read_text() + except OSError: + profile_md = None + + return RepoProfileStatus( + repo_url=repo.canonical_url, + status=repo.profile_status, + profiled_at=repo.profiled_at, + last_profiled_sha=repo.last_profiled_sha, + profile_md=profile_md, + ) + + +@router.post("/profile/rebuild", response_model=RebuildResponse) +async def rebuild_profile( + body: RebuildBody, http_request: FastAPIRequest, db=Depends(get_db) +) -> RebuildResponse: + """Schedule a re-profile of the (current or given) repo.""" + url = await _resolve_repo_url(db, body.repo_url) + if url is None: + raise HTTPException( + status_code=422, detail="No repository to profile — connect one first." + ) + task = schedule_profile_build(http_request.app, db, url) + if task is None: + return RebuildResponse(status="skipped", repo_url=url, reason="no_ai_provider") + return RebuildResponse(status="scheduled", repo_url=url) diff --git a/backend/cliff/main.py b/backend/cliff/main.py index 099c70c6..021f5340 100644 --- a/backend/cliff/main.py +++ b/backend/cliff/main.py @@ -30,6 +30,7 @@ messages, onboarding, posture, + repos, seed, sidebar, version, @@ -414,6 +415,7 @@ async def _assessment_watchdog_loop() -> None: app.include_router(assessment.router, prefix="/api") app.include_router(dashboard.router, prefix="/api") app.include_router(posture.router, prefix="/api") +app.include_router(repos.router, prefix="/api") app.include_router(completion.router, prefix="/api") app.include_router(config_routes.router, prefix="/api") diff --git a/backend/tests/api/openapi_snapshot.json b/backend/tests/api/openapi_snapshot.json index 7f4d8288..060319ee 100644 --- a/backend/tests/api/openapi_snapshot.json +++ b/backend/tests/api/openapi_snapshot.json @@ -4213,6 +4213,58 @@ "title": "PushAccessDiagnoseResponse", "type": "object" }, + "RebuildBody": { + "properties": { + "repo_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Repo Url" + } + }, + "title": "RebuildBody", + "type": "object" + }, + "RebuildResponse": { + "properties": { + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + }, + "repo_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Repo Url" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "RebuildResponse", + "type": "object" + }, "RegistryEntry": { "description": "A single integration in the catalog.", "properties": { @@ -4478,6 +4530,62 @@ "title": "RepoAgentStatus", "type": "object" }, + "RepoProfileStatus": { + "properties": { + "last_profiled_sha": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Profiled Sha" + }, + "profile_md": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profile Md" + }, + "profiled_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profiled At" + }, + "repo_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Repo Url" + }, + "status": { + "default": "none", + "title": "Status", + "type": "string" + } + }, + "title": "RepoProfileStatus", + "type": "object" + }, "RunAllResponse": { "properties": { "message": { @@ -7119,6 +7227,98 @@ ] } }, + "/api/repos/profile": { + "get": { + "description": "The Project-profile status + freshness for the (current or given) repo.", + "operationId": "get_profile_api_repos_profile_get", + "parameters": [ + { + "in": "query", + "name": "repo_url", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Repo Url" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepoProfileStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Profile", + "tags": [ + "repos" + ] + } + }, + "/api/repos/profile/rebuild": { + "post": { + "description": "Schedule a re-profile of the (current or given) repo.", + "operationId": "rebuild_profile_api_repos_profile_rebuild_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RebuildBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RebuildResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Rebuild Profile", + "tags": [ + "repos" + ] + } + }, "/api/seed": { "post": { "description": "Seed demo findings. Skips if findings already exist.", diff --git a/backend/tests/test_routes_repos.py b/backend/tests/test_routes_repos.py new file mode 100644 index 00000000..c28c9742 --- /dev/null +++ b/backend/tests/test_routes_repos.py @@ -0,0 +1,59 @@ +"""Route tests for the repo Project-profile API (ADR-0053 / PRD-0009 P1.8).""" + +from __future__ import annotations + +import cliff.db.connection as dbconn + + +async def test_get_profile_none_when_no_repo(db_client): + resp = await db_client.get("/api/repos/profile") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "none" + assert body["repo_url"] is None + + +async def test_get_profile_ready(db_client, tmp_path): + from cliff.repos.dao import finish_profile, get_or_create_repo, try_begin_profile + + conn = dbconn._db + repo = await get_or_create_repo(conn, "https://github.com/acme/web") + pdir = tmp_path / "store" + pdir.mkdir() + (pdir / "PROFILE.md").write_text("# Project profile\n## profile\n- kind: service\n") + await try_begin_profile(conn, repo.id) + await finish_profile(conn, repo.id, status="ready", sha="a" * 40, profile_dir=str(pdir)) + + # Looked up by a different spelling — canonicalization resolves it. + resp = await db_client.get( + "/api/repos/profile", params={"repo_url": "git@github.com:acme/web.git"} + ) + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "ready" + assert body["repo_url"] == "https://github.com/acme/web" + assert body["last_profiled_sha"] == "a" * 40 + assert "service" in body["profile_md"] + + +async def test_get_profile_none_when_repo_unprofiled(db_client): + resp = await db_client.get( + "/api/repos/profile", params={"repo_url": "https://github.com/x/y"} + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "none" + + +async def test_rebuild_skipped_without_ai_provider(db_client): + resp = await db_client.post( + "/api/repos/profile/rebuild", json={"repo_url": "https://github.com/a/b"} + ) + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "skipped" + assert body["reason"] == "no_ai_provider" + + +async def test_rebuild_422_without_a_repo(db_client): + resp = await db_client.post("/api/repos/profile/rebuild", json={}) + assert resp.status_code == 422 From fe4ffc9ae01a734472c7758db83ccaa4bff1f974 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:05:06 +0300 Subject: [PATCH 08/34] feat(dashboard): Project profile card (ADR-0053/PRD-0009 P1.8 frontend) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repos API client + useRepoProfile/useRebuildProfile hooks, and a Cyberdeck ProjectProfileCard on the dashboard: shows what Cliff understood, freshness (built N ago · sha), a 'what Cliff understood' disclosure, and a re-profile action (polls while building). Renders nothing when there's no repo/profile. tsc + eslint clean; 3 api tests + DashboardPage regression green. Co-Authored-By: Claude Opus 4.8 --- frontend/src/api/__tests__/repos.test.tsx | 65 ++++++++ frontend/src/api/repos.ts | 59 +++++++ .../dashboard/ProjectProfileCard.tsx | 144 ++++++++++++++++++ frontend/src/pages/DashboardPage.tsx | 5 + 4 files changed, 273 insertions(+) create mode 100644 frontend/src/api/__tests__/repos.test.tsx create mode 100644 frontend/src/api/repos.ts create mode 100644 frontend/src/components/dashboard/ProjectProfileCard.tsx diff --git a/frontend/src/api/__tests__/repos.test.tsx b/frontend/src/api/__tests__/repos.test.tsx new file mode 100644 index 00000000..c04401d7 --- /dev/null +++ b/frontend/src/api/__tests__/repos.test.tsx @@ -0,0 +1,65 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { renderHook, waitFor } from '@testing-library/react' +import { http, HttpResponse } from 'msw' +import type { ReactNode } from 'react' +import { describe, expect, it } from 'vitest' +import { server } from '../../mocks/server' +import { reposApi, useRepoProfile } from '../repos' + +function wrapper() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }) + return function Wrapper({ children }: { children: ReactNode }) { + return ( + {children} + ) + } +} + +const READY = { + repo_url: 'https://github.com/acme/web', + status: 'ready', + profiled_at: '2026-06-10T00:00:00Z', + last_profiled_sha: 'abcdef1234567890', + profile_md: '# Project profile\n- **summary:** A self-hosted web service.\n', +} + +describe('reposApi', () => { + it('getProfile hits /api/repos/profile', async () => { + server.use(http.get('/api/repos/profile', () => HttpResponse.json(READY))) + const profile = await reposApi.getProfile() + expect(profile.status).toBe('ready') + expect(profile.repo_url).toContain('acme/web') + expect(profile.last_profiled_sha).toBe('abcdef1234567890') + }) + + it('rebuildProfile posts to /api/repos/profile/rebuild', async () => { + server.use( + http.post('/api/repos/profile/rebuild', () => + HttpResponse.json({ status: 'scheduled', repo_url: READY.repo_url }), + ), + ) + const res = await reposApi.rebuildProfile() + expect(res.status).toBe('scheduled') + }) +}) + +describe('useRepoProfile', () => { + it('resolves the none state', async () => { + server.use( + http.get('/api/repos/profile', () => + HttpResponse.json({ + repo_url: null, + status: 'none', + profiled_at: null, + last_profiled_sha: null, + profile_md: null, + }), + ), + ) + const { result } = renderHook(() => useRepoProfile(), { wrapper: wrapper() }) + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(result.current.data?.status).toBe('none') + }) +}) diff --git a/frontend/src/api/repos.ts b/frontend/src/api/repos.ts new file mode 100644 index 00000000..3063c348 --- /dev/null +++ b/frontend/src/api/repos.ts @@ -0,0 +1,59 @@ +/** + * Repo Project-profile fetchers + hooks (ADR-0053 / PRD-0009). + * + * Surfaces the per-repo profile freshness for the dashboard card and the + * re-profile action. The backend resolves "the current repo" from the GitHub + * integration, so the read path needs no argument. + */ + +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { request } from './client' + +export type RepoProfileStatusValue = + | 'none' + | 'building' + | 'ready' + | 'stale' + | 'error' + +export interface RepoProfileStatus { + repo_url: string | null + status: RepoProfileStatusValue + profiled_at: string | null + last_profiled_sha: string | null + profile_md: string | null +} + +export interface RebuildResponse { + status: 'scheduled' | 'skipped' + repo_url: string | null + reason?: string | null +} + +export const reposApi = { + getProfile: () => request('/api/repos/profile'), + rebuildProfile: (repoUrl?: string) => + request('/api/repos/profile/rebuild', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(repoUrl ? { repo_url: repoUrl } : {}), + }), +} + +export function useRepoProfile() { + return useQuery({ + queryKey: ['repo-profile'], + queryFn: reposApi.getProfile, + // While a profile is building, poll so the card flips to ready on its own. + refetchInterval: (query) => + query.state.data?.status === 'building' ? 2_000 : false, + }) +} + +export function useRebuildProfile() { + const qc = useQueryClient() + return useMutation({ + mutationFn: (repoUrl?: string) => reposApi.rebuildProfile(repoUrl), + onSuccess: () => qc.invalidateQueries({ queryKey: ['repo-profile'] }), + }) +} diff --git a/frontend/src/components/dashboard/ProjectProfileCard.tsx b/frontend/src/components/dashboard/ProjectProfileCard.tsx new file mode 100644 index 00000000..73809667 --- /dev/null +++ b/frontend/src/components/dashboard/ProjectProfileCard.tsx @@ -0,0 +1,144 @@ +/** + * ProjectProfileCard — the dashboard surface for the per-repo Project profile + * (ADR-0053 / PRD-0009 Story 1 + 4). + * + * Shows that Cliff has gotten to know the project, how fresh that understanding + * is, and a re-profile action. Cyberdeck-calm, additive — renders nothing when + * there's no repo and no profile. (UX-0009 owns the eventual polish.) + */ + +import { useRebuildProfile, useRepoProfile } from '@/api/repos' +import type { RepoProfileStatusValue } from '@/api/repos' + +const STATUS_LABEL: Record = { + none: 'Not profiled', + building: 'Getting to know it…', + ready: 'Profiled', + stale: 'Out of date', + error: "Couldn't profile", +} + +const STATUS_TONE: Record = { + none: 'var(--cd-fg-4)', + building: 'var(--cd-green)', + ready: 'var(--cd-green)', + stale: 'var(--cd-amber, #E2B441)', + error: 'var(--cd-red, #E97A8E)', +} + +function timeAgo(iso: string | null): string | null { + if (!iso) return null + const then = new Date(iso).getTime() + if (Number.isNaN(then)) return null + const secs = Math.max(0, Math.round((Date.now() - then) / 1000)) + if (secs < 60) return 'just now' + const mins = Math.round(secs / 60) + if (mins < 60) return `${mins} min ago` + const hours = Math.round(mins / 60) + if (hours < 24) return `${hours} hr ago` + const days = Math.round(hours / 24) + return `${days} day${days === 1 ? '' : 's'} ago` +} + +function summaryFrom(md: string | null): string | null { + if (!md) return null + const summary = md.match(/\*\*summary:\*\*\s*(.+)/i) + if (summary) return summary[1].trim() + const kind = md.match(/\*\*kind:\*\*\s*(.+)/i) + if (kind) return `A ${kind[1].trim().replace(/_/g, ' ')} project.` + return null +} + +export default function ProjectProfileCard() { + const { data } = useRepoProfile() + const rebuild = useRebuildProfile() + + if (!data || (data.status === 'none' && !data.repo_url)) return null + + const { status } = data + const summary = summaryFrom(data.profile_md) + const when = timeAgo(data.profiled_at) + const sha = data.last_profiled_sha?.slice(0, 7) + const busy = status === 'building' || rebuild.isPending + + const headline = + status === 'building' + ? 'Getting to know your project…' + : status === 'none' + ? 'Cliff hasn’t profiled this project yet.' + : (summary ?? 'Cliff has a profile of this project.') + + return ( +
+
+ + Project profile + + + {STATUS_LABEL[status]} + +
+ +

+ {headline} +

+ + {when && status !== 'none' ? ( +

+ Built {when} + {sha ? ` · ${sha}` : ''} +

+ ) : null} + +
+ + {rebuild.data?.status === 'skipped' ? ( + + Connect an AI provider first. + + ) : null} +
+ + {data.profile_md && status !== 'none' ? ( +
+ + What Cliff understood + + 
+            {data.profile_md}
+
+
+ ) : null} +
+ ) +} diff --git a/frontend/src/pages/DashboardPage.tsx b/frontend/src/pages/DashboardPage.tsx index 8f0d8236..867101fa 100644 --- a/frontend/src/pages/DashboardPage.tsx +++ b/frontend/src/pages/DashboardPage.tsx @@ -35,6 +35,7 @@ import IssueGradeHero, { type GradeLetter, } from '@/components/dashboard/IssueGradeHero' import LastAssessmentPanel from '@/components/dashboard/LastAssessmentPanel' +import ProjectProfileCard from '@/components/dashboard/ProjectProfileCard' import type { ScannerRowData } from '@/components/dashboard/ScannerRow' import LevelUpPanel from '@/components/dashboard/LevelUpPanel' import ShareReportPanel from '@/components/dashboard/ShareReportPanel' @@ -632,6 +633,10 @@ function ReportCard({ data }: { data: DashboardPayload }) { /> ) : null} + +
+ +
) From b615d254337665dfb0d24ca5e57204ef216d3e87 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:19:42 +0300 Subject: [PATCH 09/34] feat(triage): deep-dive schemas + additive TriageOutput blocks (ADR-0052 P2.9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FindingFacts / RuleOutResult / DeepReachability (the per-finding trail) + ExploitPlan / Challenge / TriageProvenance (additive TriageOutput blocks, all optional → shipped UI + persisted rows unaffected). KillClass enum, specific Disproof. 8 tests; backward-compat verified. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/schemas.py | 175 ++++++++++++++++++ .../tests/agents/test_deep_dive_schemas.py | 102 ++++++++++ 2 files changed, 277 insertions(+) create mode 100644 backend/tests/agents/test_deep_dive_schemas.py diff --git a/backend/cliff/agents/schemas.py b/backend/cliff/agents/schemas.py index a0dee63a..d515d4a9 100644 --- a/backend/cliff/agents/schemas.py +++ b/backend/cliff/agents/schemas.py @@ -218,11 +218,92 @@ class TriageCheck(BaseModel): model_config = {"extra": "allow"} +# --------------------------------------------------------------------------- +# Deep dive (ADR-0052) — the agentic triage blocks added to TriageOutput. +# Additive: every field defaults, so a pre-deep-dive TriageOutput still loads. +# --------------------------------------------------------------------------- + + +class ReproRecipe(BaseModel): + """How V2 would reproduce the exploit. Authored in V1, executed in V2 — + the frozen sandbox seam (ADR-0052 §3).""" + + setup: list[str] = Field(default_factory=list) + docker_compose: str | None = None + image: str | None = None + ports: list[int] = Field(default_factory=list) + trigger: list[str] = Field(default_factory=list) + expected_observation: str | None = None + + model_config = {"extra": "allow"} + + +class ExploitHypothesis(BaseModel): + id: str + trigger_condition: str + attacker_input: str | None = None + reached_sink: str | None = None # file:line + expected_impact: str | None = None + impact_class: str | None = None # RCE | SSRF | SQLi | ... + repro_recipe: ReproRecipe | None = None + confidence: float = Field(default=0.0, ge=0.0, le=1.0) + + model_config = {"extra": "allow"} + + +class ExploitPlan(BaseModel): + """Ranked exploit hypotheses + repro recipes. ``no_credible_exploit`` is the + reachable-but-not-exploitable (hardening) signal (ADR-0052 §2).""" + + hypotheses: list[ExploitHypothesis] = Field(default_factory=list) + primary_hypothesis_id: str | None = None + no_credible_exploit: bool = False + + model_config = {"extra": "allow"} + + +class ChallengeReviewer(BaseModel): + lens: str # reachability | exploit | impact + verdict: Literal["holds", "refuted"] + refutation: str | None = None + + model_config = {"extra": "allow"} + + +class Challenge(BaseModel): + """The adversarial disprove panel's verdict (ADR-0052 §2). Resolution is + deterministic: a majority of ``refuted`` reviewers downgrades the verdict.""" + + verdict_holds: bool + reviewers: list[ChallengeReviewer] = Field(default_factory=list) + downgraded_verdict: TriageVerdict | None = None + confidence_adjustment: float = 0.0 + + model_config = {"extra": "allow"} + + +class TriageProvenance(BaseModel): + """What the Deep dive actually did — transparency + the SHA a verdict is + valid for (ADR-0052 §2/§6).""" + + steps_run: list[str] = Field(default_factory=list) + traced_sha: str | None = None + model_tiers: dict[str, str] = Field(default_factory=dict) # step -> tier + exit_stage: str | None = None + escalated: bool = False + + model_config = {"extra": "allow"} + + class TriageOutput(BaseModel): """The triage verdict (ADR-0051 §2). Emitted by both the deterministic scanner ``triage_synthesizer`` and the LLM ``report_triager``; the ``report`` block is populated only for reports. + The Deep dive (ADR-0052) additionally fills ``exploit_plan`` / ``challenge`` + / ``provenance`` — all optional, so the shipped UI and any pre-deep-dive + output keep working unchanged. + ``recommended_close`` is a coherent projection of ``verdict``: it is filled from the verdict when omitted and rejected when it contradicts the verdict, so the pairing is a HARD invariant (ADR-0051 §2 / eval @@ -235,6 +316,10 @@ class TriageOutput(BaseModel): exploitability: TriageExploitability | None = None report: TriageReport | None = None checks: list[TriageCheck] = Field(default_factory=list) + # Deep dive (ADR-0052) — additive, all optional. + exploit_plan: ExploitPlan | None = None + challenge: Challenge | None = None + provenance: TriageProvenance | None = None model_config = {"extra": "allow"} @@ -252,6 +337,96 @@ def _coherent_recommended_close(self) -> TriageOutput: return self +# --------------------------------------------------------------------------- +# Deep dive stage artifacts (ADR-0052 §2) — the per-finding triage trail. +# These persist to the workspace (context/triage/*.json) and feed later stages. +# --------------------------------------------------------------------------- + + +class RootCauseCandidate(BaseModel): + file: str + line: int | None = None + confidence: float = Field(default=0.0, ge=0.0, le=1.0) + why: str | None = None + + model_config = {"extra": "allow"} + + +class FindingFacts(BaseModel): + """``gather_facts`` output — the root cause pinned in *this* repo, collected + once so later stages don't re-locate it (ADR-0052 §2).""" + + vuln_class: str | None = None + root_cause_candidates: list[RootCauseCandidate] = Field(default_factory=list) + entry_point_hypothesis: str | None = None + provided_poc: str | None = None + claim: str | None = None # reports: the extracted claim + static_evidence: list[str] = Field(default_factory=list) + + model_config = {"extra": "allow"} + + +#: Why a finding was ruled out (ADR-0052 §2 / ADR-0049 FP-class catalog). +KillClass = Literal[ + "root_cause_in_nonship_code", + "dispatcher_gate", + "downstream_filter", + "production_default_off", + "duplicate_of_known", + "walk_parallel_guard", + "walk_catch_frame", + "not_reachable_by_design", + "other", +] + + +class RuleOutResult(BaseModel): + """``rule_out`` output. ``killed`` short-circuits the pipeline to the + recommended close (ADR-0052 §2 fail-cheap exit).""" + + killed: bool + kill_class: KillClass | None = None + kill_evidence: str | None = None # file:line / reference + dedup_match: str | None = None # prior issue id + surviving_concerns: list[str] = Field(default_factory=list) + recommended_verdict_on_kill: TriageClose | None = None + + model_config = {"extra": "allow"} + + +class ReachNode(BaseModel): + file: str + line: int | None = None + symbol: str | None = None + role: Literal["source", "hop", "sink"] = "hop" + note: str | None = None + + model_config = {"extra": "allow"} + + +class Disproof(BaseModel): + """The specific reason a finding is unreachable — a guard at file:line, not + an absence of evidence (ADR-0052 §2: ``unexploitable`` means *this*).""" + + guard_location: str # file:line + guard_kind: str | None = None + explanation: str | None = None + + model_config = {"extra": "allow"} + + +class DeepReachability(BaseModel): + """``trace_path`` output — the proven path, or the specific disproof.""" + + reached: Literal["yes", "no", "unknown"] + path: list[ReachNode] = Field(default_factory=list) + disproof: Disproof | None = None + trust_boundary_crossed: bool | None = None + disciplines_applied: list[str] = Field(default_factory=list) + + model_config = {"extra": "allow"} + + # Maps agent_type -> the Pydantic model for its structured_output. AGENT_OUTPUT_SCHEMAS: dict[str, type[BaseModel]] = { "finding_enricher": EnrichmentOutput, diff --git a/backend/tests/agents/test_deep_dive_schemas.py b/backend/tests/agents/test_deep_dive_schemas.py new file mode 100644 index 00000000..6a3c859f --- /dev/null +++ b/backend/tests/agents/test_deep_dive_schemas.py @@ -0,0 +1,102 @@ +"""Deep dive schema contracts (ADR-0052 §2) — additive over TriageOutput.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from cliff.agents.schemas import ( + Challenge, + ChallengeReviewer, + DeepReachability, + Disproof, + ExploitHypothesis, + ExploitPlan, + FindingFacts, + ReproRecipe, + RuleOutResult, + TriageOutput, + TriageProvenance, +) + + +def test_triage_output_backward_compatible(): + """A pre-deep-dive verdict (the shipped shape) still loads; the new blocks + default to None so the UI and persisted rows are unaffected.""" + out = TriageOutput.model_validate({"verdict": "real", "confidence": 0.9}) + assert out.exploit_plan is None + assert out.challenge is None + assert out.provenance is None + # The ADR-0051 pairing invariant still holds. + assert out.recommended_close is None + + +def test_triage_output_with_deep_dive_blocks(): + out = TriageOutput( + verdict="real", + confidence=0.88, + exploit_plan=ExploitPlan( + hypotheses=[ + ExploitHypothesis( + id="h1", + trigger_condition="unauth POST /import", + reached_sink="app/import.py:42", + impact_class="RCE", + repro_recipe=ReproRecipe(trigger=["curl ..."]), + confidence=0.7, + ) + ], + primary_hypothesis_id="h1", + ), + challenge=Challenge( + verdict_holds=True, + reviewers=[ChallengeReviewer(lens="reachability", verdict="holds")], + ), + provenance=TriageProvenance(steps_run=["gather_facts", "trace_path"], traced_sha="abc"), + ) + # round-trips + again = TriageOutput.model_validate(out.model_dump()) + assert again.exploit_plan.primary_hypothesis_id == "h1" + assert again.provenance.traced_sha == "abc" + assert again.challenge.verdict_holds is True + + +def test_exploit_plan_defaults_empty(): + plan = ExploitPlan() + assert plan.hypotheses == [] + assert plan.no_credible_exploit is False + + +def test_rule_out_kill(): + r = RuleOutResult( + killed=True, + kill_class="root_cause_in_nonship_code", + kill_evidence="tests/fixtures/x.py:3", + recommended_verdict_on_kill="false_positive", + ) + assert r.killed + assert r.recommended_verdict_on_kill == "false_positive" + + +def test_rule_out_rejects_bad_kill_class(): + with pytest.raises(ValidationError): + RuleOutResult(killed=True, kill_class="totally-made-up") + + +def test_deep_reachability_reached_enum(): + reach = DeepReachability(reached="no", disproof=Disproof(guard_location="auth.py:10")) + assert reach.reached == "no" + assert reach.disproof.guard_location == "auth.py:10" + with pytest.raises(ValidationError): + DeepReachability(reached="maybe") + + +def test_disproof_requires_guard_location(): + with pytest.raises(ValidationError): + Disproof() + + +def test_finding_facts_defaults(): + facts = FindingFacts(vuln_class="ssti") + assert facts.root_cause_candidates == [] + assert facts.static_evidence == [] From b2243ba590ab3cebd9528ab38b505adb18df82a9 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:22:11 +0300 Subject: [PATCH 10/34] feat(triage): grep tool + model tiering + escalation gate (ADR-0052 P2.1/2.2/2.3) Read-only pure-Python grep over the clone (no shell, path-scoped, skips .git); model tiering deriving cheap/strong/judge from one provider's lineup (amends ADR-0037, judge out-ranks strong per ADR-0050); escalation gate + per-assessment deep-dive budget where severity ALONE never escalates (the cost control that keeps a flood cheap). 17 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/runtime/model_tiers.py | 54 +++++++++++++ backend/cliff/agents/runtime/tools/grep.py | 81 +++++++++++++++++++ backend/cliff/agents/triage_deep/__init__.py | 6 ++ .../cliff/agents/triage_deep/escalation.py | 64 +++++++++++++++ backend/tests/agents/test_escalation.py | 43 ++++++++++ backend/tests/agents/test_grep_tool.py | 50 ++++++++++++ backend/tests/agents/test_model_tiers.py | 39 +++++++++ 7 files changed, 337 insertions(+) create mode 100644 backend/cliff/agents/runtime/model_tiers.py create mode 100644 backend/cliff/agents/runtime/tools/grep.py create mode 100644 backend/cliff/agents/triage_deep/__init__.py create mode 100644 backend/cliff/agents/triage_deep/escalation.py create mode 100644 backend/tests/agents/test_escalation.py create mode 100644 backend/tests/agents/test_grep_tool.py create mode 100644 backend/tests/agents/test_model_tiers.py diff --git a/backend/cliff/agents/runtime/model_tiers.py b/backend/cliff/agents/runtime/model_tiers.py new file mode 100644 index 00000000..8524e78f --- /dev/null +++ b/backend/cliff/agents/runtime/model_tiers.py @@ -0,0 +1,54 @@ +"""Deep dive model tiering (ADR-0052 §4, amends ADR-0037). + +ADR-0037 keeps one canonical provider + one model. The Deep dive needs three +tiers — cheap (volume), strong (the moat: reachability + exploit planning), and +judge (the adversarial challenge, which must out-rank the generator per +ADR-0050's anti-self-preference rule). We *derive* the tier map from the +configured provider's own lineup, so there's still one credential. + +Thin-lineup providers (ollama / custom) fall back to the single configured model +for every tier — the judge is then not independent (the caller logs that). +""" + +from __future__ import annotations + +#: (cheap, strong, judge) model ids per provider, without the provider prefix. +_LINEUP: dict[str, tuple[str, str, str]] = { + "anthropic": ("claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-8"), + "openrouter": ( + "anthropic/claude-haiku-4.5", + "anthropic/claude-sonnet-4.6", + "anthropic/claude-opus-4.8", + ), + "openai": ("gpt-5-mini", "gpt-5", "gpt-5"), + "google": ("gemini-2.5-flash", "gemini-2.5-pro", "gemini-2.5-pro"), +} + +TIERS = ("cheap", "strong", "judge") + + +def resolve_tier_model_ids(model_full_id: str) -> dict[str, str]: + """Derive ``{cheap, strong, judge}`` ``/`` ids. + + Falls back to the single configured model for every tier when the provider + has no known lineup (ollama / custom) or the id isn't ``provider/model``. + """ + provider, sep, _ = model_full_id.partition("/") + lineup = _LINEUP.get(provider) + if not sep or lineup is None: + return {tier: model_full_id for tier in TIERS} + cheap, strong, judge = lineup + return { + "cheap": f"{provider}/{cheap}", + "strong": f"{provider}/{strong}", + "judge": f"{provider}/{judge}", + } + + +def judge_is_independent(model_full_id: str) -> bool: + """True when the judge tier out-ranks the strong tier (a real second opinion).""" + ids = resolve_tier_model_ids(model_full_id) + return ids["judge"] != ids["strong"] + + +__all__ = ["TIERS", "judge_is_independent", "resolve_tier_model_ids"] diff --git a/backend/cliff/agents/runtime/tools/grep.py b/backend/cliff/agents/runtime/tools/grep.py new file mode 100644 index 00000000..9b870b09 --- /dev/null +++ b/backend/cliff/agents/runtime/tools/grep.py @@ -0,0 +1,81 @@ +"""``grep`` tool — read-only regex search over the workspace (ADR-0052 §3). + +The Deep dive's code-walkers (``trace_path`` / ``challenge``) need to find +references without a shell. This is pure-Python regex search scoped to the +workspace (the cached clone), no subprocess, so it's safe (read-only, can't +escape) and deterministic (gradeable by the eval). Auto-tier, like ``read``. +""" + +from __future__ import annotations + +import asyncio +import re +from pathlib import Path + +from pydantic_ai import RunContext + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools.permissions import escapes_workspace + +_MAX_MATCHES = 100 +_MAX_FILE_BYTES = 1024 * 1024 +_SKIP_DIRS = frozenset( + {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".mypy_cache"} +) + + +def search_workspace(workspace_dir: str, pattern: str, path: str = ".") -> str: + """Search files under *workspace_dir*/*path* for *pattern* (regex). + + Returns up to 100 ``relpath:line: text`` matches, or a bracketed status + ([refused] / [invalid regex] / [no matches]). Pure + synchronous so it's + unit-testable without a RunContext; the tool wraps it on a thread. + """ + if escapes_workspace(workspace_dir, path): + return f"[refused: {path} resolves outside the workspace]" + try: + rx = re.compile(pattern) + except re.error as exc: + return f"[invalid regex: {exc}]" + + root = Path(workspace_dir) + base = (root / path).resolve() + if not base.exists(): + return f"[path not found: {path}]" + + matches: list[str] = [] + candidates = [base] if base.is_file() else base.rglob("*") + for f in candidates: + if len(matches) >= _MAX_MATCHES: + break + if not f.is_file() or any(part in _SKIP_DIRS for part in f.parts): + continue + try: + if f.stat().st_size > _MAX_FILE_BYTES: + continue + with f.open("r", encoding="utf-8", errors="replace") as fh: + for lineno, line in enumerate(fh, 1): + if rx.search(line): + rel = f.relative_to(root) + matches.append(f"{rel}:{lineno}: {line.rstrip()[:200]}") + if len(matches) >= _MAX_MATCHES: + break + except OSError: + continue + + if not matches: + return f"[no matches for {pattern!r}]" + out = "\n".join(matches) + if len(matches) >= _MAX_MATCHES: + out += f"\n[... capped at {_MAX_MATCHES} matches ...]" + return out + + +async def grep(ctx: RunContext[WorkspaceDeps], pattern: str, path: str = ".") -> str: + """Search the workspace for *pattern* (regex), under *path* (default: root).""" + return await asyncio.to_thread( + search_workspace, ctx.deps.workspace_dir, pattern, path + ) + + +__all__ = ["grep", "search_workspace"] diff --git a/backend/cliff/agents/triage_deep/__init__.py b/backend/cliff/agents/triage_deep/__init__.py new file mode 100644 index 00000000..44cfff4a --- /dev/null +++ b/backend/cliff/agents/triage_deep/__init__.py @@ -0,0 +1,6 @@ +"""The agentic triage Deep dive (ADR-0052). + +The escalation-gated pipeline that investigates uncertain / high-stakes findings +by reading real source — gather_facts -> rule_out -> trace_path -> plan_exploit +-> challenge — read-only and plan-don't-execute in V1. +""" diff --git a/backend/cliff/agents/triage_deep/escalation.py b/backend/cliff/agents/triage_deep/escalation.py new file mode 100644 index 00000000..d138ccc5 --- /dev/null +++ b/backend/cliff/agents/triage_deep/escalation.py @@ -0,0 +1,64 @@ +"""The escalation gate + deep-dive budget (ADR-0052 §1). + +Decides whether a Quick-read verdict is worth a Deep dive. The load-bearing cost +control: severity *alone* never escalates — it needs uncertainty +(``needs_review``) or stakes-plus-remaining-budget — so a flood of high-severity +dependency CVEs can't quietly turn the cheap default into an all-deep-dive run. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +#: Deep dives allowed per assessment before high-stakes findings queue for an +#: explicit user-triggered dive (eval-tuned, not frozen). +DEFAULT_DEEP_DIVE_BUDGET = 10 + + +@dataclass(frozen=True) +class EscalationDecision: + escalate: bool + reason: str + + +def is_high_stakes(finding: dict) -> bool: + """High/critical severity, internet-facing, or a crown-jewel asset.""" + severity = str(finding.get("raw_severity") or finding.get("severity") or "").lower() + if severity in ("critical", "high"): + return True + if finding.get("internet_facing") is True: + return True + return bool(finding.get("crown_jewel")) + + +def decide_escalation( + verdict: str, + finding: dict, + *, + budget_remaining: int, + source: str = "scanner", +) -> EscalationDecision: + """Whether to run the Deep dive after the Quick read. + + Reports always escalate (no cheap projection to trust). Otherwise: escalate + on uncertainty always; on stakes only while deep-dive budget remains. + """ + if source == "report": + return EscalationDecision(True, "report — always deep dive") + if verdict == "needs_review": + return EscalationDecision(True, "uncertain (needs_review)") + if is_high_stakes(finding): + if budget_remaining > 0: + return EscalationDecision(True, "high-stakes — produce evidence") + return EscalationDecision( + False, "high-stakes but deep-dive budget exhausted — queue for manual" + ) + return EscalationDecision(False, "clear and low-stakes — cheap verdict stands") + + +__all__ = [ + "DEFAULT_DEEP_DIVE_BUDGET", + "EscalationDecision", + "decide_escalation", + "is_high_stakes", +] diff --git a/backend/tests/agents/test_escalation.py b/backend/tests/agents/test_escalation.py new file mode 100644 index 00000000..0d46fde4 --- /dev/null +++ b/backend/tests/agents/test_escalation.py @@ -0,0 +1,43 @@ +"""Unit tests for the escalation gate + budget (ADR-0052 §1).""" + +from __future__ import annotations + +from cliff.agents.triage_deep.escalation import decide_escalation, is_high_stakes + +LOW = {"raw_severity": "low"} +CRIT = {"raw_severity": "critical"} +NET = {"raw_severity": "medium", "internet_facing": True} + + +def test_needs_review_always_escalates(): + d = decide_escalation("needs_review", LOW, budget_remaining=0) + assert d.escalate is True + + +def test_clear_low_stakes_does_not_escalate(): + d = decide_escalation("unexploitable", LOW, budget_remaining=10) + assert d.escalate is False + + +def test_high_stakes_escalates_with_budget(): + d = decide_escalation("real", CRIT, budget_remaining=3) + assert d.escalate is True + assert "high-stakes" in d.reason + + +def test_severity_alone_does_not_escalate_without_budget(): + # The load-bearing cost control: a flood of high-severity findings can't + # all deep-dive once the per-assessment budget is spent. + d = decide_escalation("real", CRIT, budget_remaining=0) + assert d.escalate is False + assert "budget" in d.reason + + +def test_reports_always_escalate(): + d = decide_escalation("real", LOW, budget_remaining=0, source="report") + assert d.escalate is True + + +def test_internet_facing_is_high_stakes(): + assert is_high_stakes(NET) is True + assert is_high_stakes(LOW) is False diff --git a/backend/tests/agents/test_grep_tool.py b/backend/tests/agents/test_grep_tool.py new file mode 100644 index 00000000..0c9ccf1a --- /dev/null +++ b/backend/tests/agents/test_grep_tool.py @@ -0,0 +1,50 @@ +"""Unit tests for the read-only grep tool (ADR-0052 §3).""" + +from __future__ import annotations + +import pytest + +from cliff.agents.runtime.tools.grep import search_workspace + + +@pytest.fixture +def ws(tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "main.py").write_text("def run():\n eval(user_input)\n") + (tmp_path / "app" / "safe.py").write_text("x = 1\n") + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "config").write_text("eval secret\n") + return tmp_path + + +def test_finds_matches_with_file_line(ws): + out = search_workspace(str(ws), r"eval\(") + assert "app/main.py:2:" in out + assert "eval(user_input)" in out + + +def test_no_matches(ws): + out = search_workspace(str(ws), "nonexistent_symbol") + assert "[no matches" in out + + +def test_skips_dot_git(ws): + # The .git/config line also contains "eval" but must be skipped. + out = search_workspace(str(ws), "eval") + assert ".git" not in out + assert "app/main.py" in out + + +def test_invalid_regex_is_reported(ws): + out = search_workspace(str(ws), "(unclosed") + assert "[invalid regex" in out + + +def test_path_escape_refused(ws): + out = search_workspace(str(ws), "x", path="../../../etc") + assert "[refused" in out + + +def test_scoped_path(ws): + out = search_workspace(str(ws), "x = 1", path="app") + assert "app/safe.py" in out diff --git a/backend/tests/agents/test_model_tiers.py b/backend/tests/agents/test_model_tiers.py new file mode 100644 index 00000000..ad20f4ea --- /dev/null +++ b/backend/tests/agents/test_model_tiers.py @@ -0,0 +1,39 @@ +"""Unit tests for Deep dive model tiering (ADR-0052 §4).""" + +from __future__ import annotations + +from cliff.agents.runtime.model_tiers import ( + judge_is_independent, + resolve_tier_model_ids, +) + + +def test_anthropic_lineup_has_three_distinct_tiers(): + ids = resolve_tier_model_ids("anthropic/claude-haiku-4-5") + assert ids["cheap"] == "anthropic/claude-haiku-4-5" + assert ids["strong"] == "anthropic/claude-sonnet-4-6" + assert ids["judge"] == "anthropic/claude-opus-4-8" + # judge out-ranks strong → a real second opinion. + assert ids["cheap"] != ids["strong"] != ids["judge"] + + +def test_openrouter_keeps_provider_prefix(): + ids = resolve_tier_model_ids("openrouter/anthropic/claude-haiku-4.5") + assert ids["cheap"].startswith("openrouter/anthropic/") + assert ids["judge"].startswith("openrouter/anthropic/") + assert ids["judge"] != ids["strong"] + + +def test_unknown_provider_falls_back_to_single_model(): + ids = resolve_tier_model_ids("ollama/llama3") + assert ids["cheap"] == ids["strong"] == ids["judge"] == "ollama/llama3" + + +def test_no_slash_falls_back(): + ids = resolve_tier_model_ids("weird-id") + assert set(ids.values()) == {"weird-id"} + + +def test_judge_independence_signal(): + assert judge_is_independent("anthropic/claude-haiku-4-5") is True + assert judge_is_independent("ollama/llama3") is False From 686b791cb19b605c4bf0cbf63a11d7277951591f Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:24:59 +0300 Subject: [PATCH 11/34] feat(triage): the five Deep dive agents + adversarial challenge panel (ADR-0052 P2.4-2.8) gather_facts/rule_out/trace_path/plan_exploit (read-only, read+grep over the clone, walk-* + FP-class disciplines in the prompts) + the challenge panel (one adversarial reviewer per lens, DETERMINISTIC majority-refutes resolution that downgrades to needs_review). Exported grep from the tools package. TestModel- driven; 10 tests. Co-Authored-By: Claude Opus 4.8 --- .../cliff/agents/runtime/tools/__init__.py | 2 + backend/cliff/agents/triage_deep/agents.py | 165 ++++++++++++++++++ backend/cliff/agents/triage_deep/challenge.py | 116 ++++++++++++ backend/tests/agents/test_deep_dive_agents.py | 117 +++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 backend/cliff/agents/triage_deep/agents.py create mode 100644 backend/cliff/agents/triage_deep/challenge.py create mode 100644 backend/tests/agents/test_deep_dive_agents.py diff --git a/backend/cliff/agents/runtime/tools/__init__.py b/backend/cliff/agents/runtime/tools/__init__.py index 182bddb3..3f2aa4c4 100644 --- a/backend/cliff/agents/runtime/tools/__init__.py +++ b/backend/cliff/agents/runtime/tools/__init__.py @@ -14,6 +14,7 @@ from cliff.agents.runtime.tools.bash import bash from cliff.agents.runtime.tools.edit import edit from cliff.agents.runtime.tools.gh import gh +from cliff.agents.runtime.tools.grep import grep from cliff.agents.runtime.tools.permissions import ( classify_tool_request, gate_tool_call, @@ -32,6 +33,7 @@ "edit", "gate_tool_call", "gh", + "grep", "read", "webfetch", ] diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py new file mode 100644 index 00000000..b24e5de7 --- /dev/null +++ b/backend/cliff/agents/triage_deep/agents.py @@ -0,0 +1,165 @@ +"""The Deep dive's four linear-stage agents (ADR-0052 §2). + +gather_facts -> rule_out -> trace_path -> plan_exploit. (The fifth stage, +``challenge``, is an adversarial panel — see ``challenge.py``.) All are +in-process Pydantic AI agents (ADR-0047), **read-only** (read + grep over the +cached clone, nothing else), driven by TestModel/FunctionModel in CI. Verdict +quality is the key-gated eval. + +Deps reuse (documented, as for the profile builders): ``WorkspaceDeps`` with +``workspace_dir`` = the repo clone and the prior stage artifacts + repo +knowledge threaded through ``prior_context`` + the rendered prompt. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from pydantic_ai import Agent +from pydantic_ai.usage import UsageLimits + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools.grep import grep +from cliff.agents.runtime.tools.read import read +from cliff.agents.schemas import ( + DeepReachability, + ExploitPlan, + FindingFacts, + RuleOutResult, +) + +if TYPE_CHECKING: + from pydantic_ai.models import Model + +#: The Deep dive's COMPLETE tool surface — read-only (ADR-0052 §3). One constant +#: the trust-boundary test asserts on. +DEEP_DIVE_TOOLS = (read, grep) + +#: trace_path / challenge can read several files; cap so a weak model can't loop. +DEEP_DIVE_REQUEST_LIMIT = 25 + +GATHER_PROMPT = """\ +You are pinning down a vulnerability finding in THIS repository. Using `read` \ +and `grep`, locate the root cause in this repo's actual code (file:line \ +candidates), identify the vulnerability class, and state the entry-point \ +hypothesis. Collect the static evidence later stages will reuse so they don't \ +re-locate it. For an inbound report, also extract the reporter's claim. Read \ +sparingly — you do not need every file.""" + +RULE_OUT_PROMPT = """\ +Decide whether this finding can be ruled out cheaply, BEFORE any expensive \ +reachability analysis. Work through the false-positive catalog: +- root cause only in test / fixture / example / vendored / dead code (use the \ +code map) -> kill_class=root_cause_in_nonship_code +- a dispatcher-level gate or a downstream consumer that filters the input \ +-> dispatcher_gate / downstream_filter +- gated behind a production-default-off flag -> production_default_off +- a duplicate of a known/fixed issue (use the threat history) -> duplicate_of_known +- a sibling site has the guard (walk-the-parallel-guard) or a surrounding \ +catch frame (walk-the-catch-frame) +- not reachable by design -> not_reachable_by_design +If a kill applies, set killed=true with the kill_class and file:line evidence, \ +and recommend `false_positive` (not a real issue) or `unexploitable` (real \ +advisory, not reachable here). Otherwise killed=false and list the surviving \ +concerns. When unsure, do NOT kill.""" + +TRACE_PROMPT = """\ +Determine whether an attacker can actually REACH the vulnerable code. Walk from \ +a user-controlled entry point to the sink with file:line at EVERY hop, using \ +`read` and `grep`. Apply these disciplines and record which you used: +- walk-the-catch-frame (a surrounding catch neutralizes it) +- walk-the-parallel-guard (a sibling site / the entry function holds the guard) +- walk-the-downstream-gate (a consumer downstream validates the input) +- runtime-overrides-summarized-source (verify the code is reachable in the real \ +build, not behind a disabled flag) +- tail-call-vs-post-call +If reachable, return reached=yes with the path (source -> hops -> sink). If a \ +SPECIFIC guard blocks it, return reached=no with the disproof (the guard's \ +file:line and why). If undeterminable, reached=unknown. AI confidence is not a \ +vulnerability — no hop without a file:line you actually read.""" + +PLAN_PROMPT = """\ +The vulnerability is reachable. Lay out how it could be exploited: ranked \ +hypotheses, each with the trigger condition, the attacker input that reaches \ +the sink (file:line), the expected impact and impact_class, and a docker repro \ +recipe (setup steps, trigger, expected observation). DO NOT run anything — \ +author the plan only; it is a plan, not a demonstrated exploit. If despite \ +reachability there is no credible exploit (an architectural gap, hardening not \ +a vuln), set no_credible_exploit=true.""" + + +def _agent(model: Model, output_type: type, system_prompt: str) -> Agent: + return Agent( + model=model, + output_type=output_type, + deps_type=WorkspaceDeps, + system_prompt=system_prompt, + tools=list(DEEP_DIVE_TOOLS), + ) + + +def build_gather_facts_agent(model: Model) -> Agent[WorkspaceDeps, FindingFacts]: + return _agent(model, FindingFacts, GATHER_PROMPT) + + +def build_rule_out_agent(model: Model) -> Agent[WorkspaceDeps, RuleOutResult]: + return _agent(model, RuleOutResult, RULE_OUT_PROMPT) + + +def build_trace_path_agent(model: Model) -> Agent[WorkspaceDeps, DeepReachability]: + return _agent(model, DeepReachability, TRACE_PROMPT) + + +def build_plan_exploit_agent(model: Model) -> Agent[WorkspaceDeps, ExploitPlan]: + return _agent(model, ExploitPlan, PLAN_PROMPT) + + +def render_context(deps: WorkspaceDeps) -> str: + """The data blob handed to a stage agent: the finding + the prior artifacts / + repo knowledge it declared in ``prior_context``.""" + parts = [f"## Finding\n{json.dumps(deps.finding, indent=2, default=str)}"] + for key, value in deps.prior_context.items(): + if value: + parts.append(f"## {key}\n{json.dumps(value, indent=2, default=str)}") + return "\n\n".join(parts) + + +async def _run(agent: Agent, deps: WorkspaceDeps) -> dict: + result = await agent.run( + render_context(deps), + deps=deps, + usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT), + ) + return result.output.model_dump() + + +async def run_gather_facts(deps: WorkspaceDeps, model: Model) -> dict: + return await _run(build_gather_facts_agent(model), deps) + + +async def run_rule_out(deps: WorkspaceDeps, model: Model) -> dict: + return await _run(build_rule_out_agent(model), deps) + + +async def run_trace_path(deps: WorkspaceDeps, model: Model) -> dict: + return await _run(build_trace_path_agent(model), deps) + + +async def run_plan_exploit(deps: WorkspaceDeps, model: Model) -> dict: + return await _run(build_plan_exploit_agent(model), deps) + + +__all__ = [ + "DEEP_DIVE_REQUEST_LIMIT", + "DEEP_DIVE_TOOLS", + "build_gather_facts_agent", + "build_plan_exploit_agent", + "build_rule_out_agent", + "build_trace_path_agent", + "render_context", + "run_gather_facts", + "run_plan_exploit", + "run_rule_out", + "run_trace_path", +] diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py new file mode 100644 index 00000000..17c7533f --- /dev/null +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -0,0 +1,116 @@ +"""Challenge — the adversarial disprove panel (ADR-0052 §2). + +The single check standing between us and shipping a confidently-wrong verdict. +A small fixed panel of reviewers, each on a distinct lens, each tasked to BREAK +the verdict, on the judge tier (a stronger model than the generator, ADR-0050). +Resolution is DETERMINISTIC (review finding #4): a majority of ``refuted`` +reviewers downgrades the verdict; a tie holds but caps confidence. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from pydantic_ai import Agent +from pydantic_ai.usage import UsageLimits + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools.grep import grep +from cliff.agents.runtime.tools.read import read +from cliff.agents.schemas import Challenge, ChallengeReviewer +from cliff.agents.triage_deep.agents import DEEP_DIVE_REQUEST_LIMIT, render_context + +if TYPE_CHECKING: + from pydantic_ai.models import Model + +#: One reviewer per lens — diversity catches failure modes redundancy can't. +CHALLENGE_LENSES: dict[str, str] = { + "reachability": ( + "Attack the reachability path. Is there a guard the tracer missed " + "(walk-the-parallel-guard)? Is a hop actually unreachable in the real build?" + ), + "exploit": ( + "Attack the exploit plan. Would the trigger actually fire? Is the input " + "genuinely attacker-controlled, or normalized/validated upstream?" + ), + "impact": ( + "Attack the impact. Is it demonstrated by the evidence or merely assumed? " + "Is the severity inflated beyond what the path supports?" + ), +} + +_SYSTEM = """\ +You are an adversarial reviewer. Your job is to try to PROVE the triage verdict \ +WRONG through one specific lens — not to confirm it. {lens_instruction} Read the \ +cited code with `read`/`grep` to check claims yourself rather than trusting the \ +summary. Apply the test: "would a tired triager reading 30 reports a day believe \ +this at the cited severity?" Default to `refuted` when the evidence is weak or \ +you cannot verify a load-bearing claim. Return holds or refuted with a concrete \ +refutation.""" + + +def build_reviewer_agent(model: Model, lens: str) -> Agent[WorkspaceDeps, ChallengeReviewer]: + return Agent( + model=model, + output_type=ChallengeReviewer, + deps_type=WorkspaceDeps, + system_prompt=_SYSTEM.format(lens_instruction=CHALLENGE_LENSES[lens]), + tools=[read, grep], + ) + + +def resolve_challenge( + reviewers: list[ChallengeReviewer], current_verdict: str +) -> Challenge: + """Deterministic resolution (ADR-0052 §2). + + Majority ``refuted`` -> downgrade to ``needs_review`` (conservative: a failed + challenge never auto-promotes to ``real``). A tie holds but caps confidence. + """ + if not reviewers: + return Challenge(verdict_holds=True, reviewers=[], confidence_adjustment=0.0) + + refuted = sum(1 for r in reviewers if r.verdict == "refuted") + holds = len(reviewers) - refuted + + if refuted > holds: + return Challenge( + verdict_holds=False, + reviewers=reviewers, + downgraded_verdict="needs_review", + confidence_adjustment=-0.25, + ) + if refuted == holds and refuted > 0: + # Tie — the verdict survives but is no longer high-confidence. + return Challenge( + verdict_holds=True, reviewers=reviewers, confidence_adjustment=-0.1 + ) + return Challenge(verdict_holds=True, reviewers=reviewers, confidence_adjustment=0.0) + + +async def run_challenge_panel( + deps: WorkspaceDeps, model: Model, current_verdict: str +) -> Challenge: + """Run every lens reviewer in parallel and resolve deterministically.""" + prompt = render_context(deps) + + async def _one(lens: str) -> ChallengeReviewer: + agent = build_reviewer_agent(model, lens) + result = await agent.run( + prompt, deps=deps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) + ) + out = result.output + # Pin the lens — the reviewer's job is fixed by construction, not its choice. + return out.model_copy(update={"lens": lens}) + + reviewers = await asyncio.gather(*(_one(lens) for lens in CHALLENGE_LENSES)) + return resolve_challenge(list(reviewers), current_verdict) + + +__all__ = [ + "CHALLENGE_LENSES", + "build_reviewer_agent", + "resolve_challenge", + "run_challenge_panel", +] diff --git a/backend/tests/agents/test_deep_dive_agents.py b/backend/tests/agents/test_deep_dive_agents.py new file mode 100644 index 00000000..18d8a8cf --- /dev/null +++ b/backend/tests/agents/test_deep_dive_agents.py @@ -0,0 +1,117 @@ +"""Deep dive agents — read-only boundary, shape, deterministic challenge. + +Keyless: TestModel drives the agents (no real LLM); quality is the key-gated +eval. These assert the trust boundary, that each stage returns its validated +artifact, and the deterministic challenge resolution. +""" + +from __future__ import annotations + +import pytest +from pydantic_ai.models.test import TestModel + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.tools import bash, edit, gh, grep, read, webfetch +from cliff.agents.schemas import ( + Challenge, + ChallengeReviewer, + DeepReachability, + ExploitPlan, + FindingFacts, + RuleOutResult, +) +from cliff.agents.triage_deep.agents import ( + DEEP_DIVE_TOOLS, + run_gather_facts, + run_plan_exploit, + run_rule_out, + run_trace_path, +) +from cliff.agents.triage_deep.challenge import ( + CHALLENGE_LENSES, + resolve_challenge, + run_challenge_panel, +) + + +def test_deep_dive_is_read_only(): + assert DEEP_DIVE_TOOLS == (read, grep) + for forbidden in (bash, edit, gh, webfetch): + assert forbidden not in DEEP_DIVE_TOOLS + + +@pytest.fixture +def deps(tmp_path): + clone = tmp_path / "repo" + clone.mkdir() + (clone / "app.py").write_text("def handler(req):\n eval(req.body)\n") + return WorkspaceDeps( + workspace_id="dd", + workspace_dir=str(clone), + finding={"source_type": "code", "title": "eval on request body"}, + prior_context={"profile": {"kind": "service"}, "code_map": {"ships_roots": ["**"]}}, + ) + + +async def test_gather_facts_returns_findingfacts(deps): + out = await run_gather_facts(deps, TestModel(custom_output_args={"vuln_class": "rce"})) + assert FindingFacts.model_validate(out).vuln_class == "rce" + + +async def test_rule_out_returns_result(deps): + out = await run_rule_out(deps, TestModel(custom_output_args={"killed": False})) + assert RuleOutResult.model_validate(out).killed is False + + +async def test_trace_path_returns_reachability(deps): + out = await run_trace_path(deps, TestModel(custom_output_args={"reached": "yes"})) + assert DeepReachability.model_validate(out).reached == "yes" + + +async def test_plan_exploit_returns_plan(deps): + out = await run_plan_exploit(deps, TestModel(custom_output_args={"no_credible_exploit": True})) + assert ExploitPlan.model_validate(out).no_credible_exploit is True + + +# ── deterministic challenge resolution ────────────────────────────────────── + + +def _rev(verdict): + return ChallengeReviewer(lens="reachability", verdict=verdict) + + +def test_resolve_all_hold(): + c = resolve_challenge([_rev("holds"), _rev("holds"), _rev("holds")], "real") + assert c.verdict_holds is True + assert c.downgraded_verdict is None + assert c.confidence_adjustment == 0.0 + + +def test_resolve_majority_refuted_downgrades(): + c = resolve_challenge([_rev("refuted"), _rev("refuted"), _rev("holds")], "real") + assert c.verdict_holds is False + assert c.downgraded_verdict == "needs_review" + assert c.confidence_adjustment < 0 + + +def test_resolve_tie_holds_but_caps_confidence(): + c = resolve_challenge([_rev("refuted"), _rev("holds")], "real") + assert c.verdict_holds is True + assert c.downgraded_verdict is None + assert c.confidence_adjustment < 0 + + +def test_resolve_empty_holds(): + c = resolve_challenge([], "real") + assert c.verdict_holds is True + + +async def test_challenge_panel_runs_all_lenses(deps): + c = await run_challenge_panel( + deps, TestModel(custom_output_args={"lens": "x", "verdict": "holds"}), "real" + ) + assert isinstance(c, Challenge) + assert c.verdict_holds is True + assert len(c.reviewers) == len(CHALLENGE_LENSES) + # The lens is pinned by construction, not the model's choice. + assert {r.lens for r in c.reviewers} == set(CHALLENGE_LENSES) From 7d0316a055db3c80cd5b8b1421f672df94de5471 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:25:18 +0300 Subject: [PATCH 12/34] style(triage): ruff autofix on deep-dive agent test Co-Authored-By: Claude Opus 4.8 --- backend/tests/agents/test_deep_dive_agents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/agents/test_deep_dive_agents.py b/backend/tests/agents/test_deep_dive_agents.py index 18d8a8cf..34cf8854 100644 --- a/backend/tests/agents/test_deep_dive_agents.py +++ b/backend/tests/agents/test_deep_dive_agents.py @@ -35,7 +35,7 @@ def test_deep_dive_is_read_only(): - assert DEEP_DIVE_TOOLS == (read, grep) + assert (read, grep) == DEEP_DIVE_TOOLS for forbidden in (bash, edit, gh, webfetch): assert forbidden not in DEEP_DIVE_TOOLS From babe17abd7f15f540a37c6c4c659de49c82032d6 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:27:28 +0300 Subject: [PATCH 13/34] feat(triage): DeepDiveRunner orchestration with fail-cheap exits (ADR-0052 P2.10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear backbone wiring the five stages with early exits (rule-out kill → false_positive; disproof → unexploitable; unknown → needs_review; reachable-no- exploit → unexploitable/hardening; challenge holds → real, refuted → downgrade) and TriageOutput assembly (mapped reachability for the shipped UI, exploit_plan, challenge, SHA-pinned provenance with per-step model tiers). Injected stages → every path TDD'd. 6 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/runner.py | 258 ++++++++++++++++++ backend/tests/agents/test_deep_dive_runner.py | 128 +++++++++ 2 files changed, 386 insertions(+) create mode 100644 backend/cliff/agents/triage_deep/runner.py create mode 100644 backend/tests/agents/test_deep_dive_runner.py diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py new file mode 100644 index 00000000..f427fc9d --- /dev/null +++ b/backend/cliff/agents/triage_deep/runner.py @@ -0,0 +1,258 @@ +"""DeepDiveRunner — orchestrates the escalation-gated Deep dive (ADR-0052 §2). + +Linear backbone with fail-cheap early exits, the two panels at their stages, and +final TriageOutput assembly. The five stages are injected (default: the real +agents), so every exit path is unit-tested without a model. Model tiers (cheap +for gather/rule_out, strong for trace/plan, judge for challenge) come from +``build_tier_models``. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.model_tiers import resolve_tier_model_ids +from cliff.agents.schemas import ( + Challenge, + ExploitPlan, + TriageCheck, + TriageExploitability, + TriageOutput, + TriageProvenance, + TriageReachability, + TriageReachabilityNode, +) +from cliff.agents.triage_deep.agents import ( + run_gather_facts, + run_plan_exploit, + run_rule_out, + run_trace_path, +) +from cliff.agents.triage_deep.challenge import run_challenge_panel + +if TYPE_CHECKING: + from pathlib import Path + + from pydantic_ai.models import Model + +_STEP_TIER = { + "gather_facts": "cheap", + "rule_out": "cheap", + "trace_path": "strong", + "plan_exploit": "strong", + "challenge": "judge", +} + +# Confidence anchors (ADR-0052; tuned against the eval, not frozen). +_CONF_KILL = 0.85 +_CONF_DISPROOF = 0.8 +_CONF_HARDENING = 0.7 +_CONF_REAL = 0.85 +_CONF_UNKNOWN = 0.5 + + +def _clamp(x: float) -> float: + return max(0.0, min(1.0, x)) + + +def _deps(clone_dir: Path | str, finding: dict, prior: dict[str, Any]) -> WorkspaceDeps: + return WorkspaceDeps( + workspace_id="triage-deep", + workspace_dir=str(clone_dir), + finding=finding, + prior_context={k: v for k, v in prior.items() if v is not None}, + ) + + +def _map_reach(reach: dict) -> TriageReachability: + """DeepReachability dict -> the TriageReachability the shipped UI renders.""" + nodes: list[TriageReachabilityNode] = [] + for n in reach.get("path", []) or []: + loc = n.get("file") or "" + if n.get("line"): + loc = f"{loc}:{n['line']}" + nodes.append( + TriageReachabilityNode( + label=n.get("symbol") or n.get("file") or "node", + detail=loc or None, + kind=n.get("role"), + ) + ) + return TriageReachability(reached=reach.get("reached") == "yes", path=nodes) + + +@dataclass(frozen=True) +class DeepDiveStages: + gather: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_gather_facts + rule_out: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_rule_out + trace: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_trace_path + plan: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_plan_exploit + challenge: Callable[[WorkspaceDeps, Any, str], Awaitable[Challenge]] = run_challenge_panel + + +def build_tier_models(env: dict[str, str], model_full_id: str) -> dict[str, Model]: + """Build the {cheap, strong, judge} Model map from the canonical AI state.""" + from cliff.agents.runtime.provider import build_model + + return { + tier: build_model(env, mid) + for tier, mid in resolve_tier_model_ids(model_full_id).items() + } + + +class DeepDiveRunner: + def __init__( + self, + models: dict[str, Any], + *, + stages: DeepDiveStages | None = None, + ) -> None: + self._models = models + self._stages = stages or DeepDiveStages() + + async def run( + self, + *, + finding: dict, + repo_knowledge: dict, + clone_dir: Path | str, + enrichment: dict | None = None, + exposure: dict | None = None, + traced_sha: str | None = None, + ) -> TriageOutput: + steps: list[str] = [] + base = { + "profile": repo_knowledge.get("profile"), + "code_map": repo_knowledge.get("code_map"), + "threat": repo_knowledge.get("threat"), + } + + def prov(exit_stage: str) -> TriageProvenance: + return TriageProvenance( + steps_run=list(steps), + traced_sha=traced_sha, + exit_stage=exit_stage, + escalated=True, + model_tiers={s: _STEP_TIER[s] for s in steps}, + ) + + # 1. Gather the facts (cheap). + facts = await self._stages.gather( + _deps(clone_dir, finding, {**base, "enrichment": enrichment, "exposure": exposure}), + self._models["cheap"], + ) + steps.append("gather_facts") + + # 2. Rule out false alarms (cheap) — fail-cheap exit. + ro = await self._stages.rule_out( + _deps(clone_dir, finding, {**base, "facts": facts}), self._models["cheap"] + ) + steps.append("rule_out") + if ro.get("killed"): + verdict = ro.get("recommended_verdict_on_kill") or "false_positive" + return TriageOutput( + verdict=verdict, + confidence=_CONF_KILL, + checks=[ + TriageCheck( + eyebrow="Ruled out", + result=ro.get("kill_class") or "false alarm", + kind="pass", + detail=ro.get("kill_evidence"), + ) + ], + provenance=prov("rule_out"), + ) + + # 3. Trace the path (strong) — fail-cheap exit on a specific disproof. + reach = await self._stages.trace( + _deps(clone_dir, finding, {**base, "facts": facts}), self._models["strong"] + ) + steps.append("trace_path") + reached = reach.get("reached") + if reached == "no": + disproof = reach.get("disproof") or {} + return TriageOutput( + verdict="unexploitable", + confidence=_CONF_DISPROOF, + reachability=TriageReachability( + reached=False, + summary=disproof.get("explanation") or "A specific guard blocks this path.", + ), + exploitability=TriageExploitability(exploitable="no", reason=disproof.get("explanation")), + checks=[ + TriageCheck( + eyebrow="Disproof", + result="Not reachable", + kind="pass", + detail=disproof.get("guard_location"), + ) + ], + provenance=prov("trace_path"), + ) + if reached == "unknown": + return TriageOutput( + verdict="needs_review", + confidence=_CONF_UNKNOWN, + reachability=_map_reach(reach), + provenance=prov("trace_path"), + ) + + # 4. Plan the exploit (strong) — reachable-but-no-exploit = hardening. + plan = await self._stages.plan( + _deps(clone_dir, finding, {**base, "facts": facts, "reachability": reach}), + self._models["strong"], + ) + steps.append("plan_exploit") + if plan.get("no_credible_exploit"): + return TriageOutput( + verdict="unexploitable", + confidence=_CONF_HARDENING, + reachability=_map_reach(reach), + exploitability=TriageExploitability( + exploitable="no", + reason="Reachable, but no credible exploit (hardening, not a vulnerability).", + ), + exploit_plan=ExploitPlan.model_validate(plan), + provenance=prov("plan_exploit"), + ) + + # 5. Challenge the verdict (judge) — adversarial, deterministic resolution. + challenge = await self._stages.challenge( + _deps( + clone_dir, + finding, + {**base, "facts": facts, "reachability": reach, "exploit_plan": plan}, + ), + self._models["judge"], + "real", + ) + steps.append("challenge") + + verdict = "real" if challenge.verdict_holds else (challenge.downgraded_verdict or "needs_review") + confidence = _clamp(_CONF_REAL + challenge.confidence_adjustment) + return TriageOutput( + verdict=verdict, + confidence=confidence, + reachability=_map_reach(reach), + exploitability=TriageExploitability( + exploitable="yes", reason="Reachable with a credible exploit path." + ), + exploit_plan=ExploitPlan.model_validate(plan), + challenge=challenge, + checks=[ + TriageCheck( + eyebrow="Challenge", + result="Held" if challenge.verdict_holds else "Downgraded", + kind="pass" if challenge.verdict_holds else "warn", + detail=f"{len(challenge.reviewers)} adversarial reviewers", + ) + ], + provenance=prov("challenge"), + ) + + +__all__ = ["DeepDiveRunner", "DeepDiveStages", "build_tier_models"] diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py new file mode 100644 index 00000000..5f8964a0 --- /dev/null +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -0,0 +1,128 @@ +"""DeepDiveRunner orchestration — every fail-cheap exit + the final assembly. + +Stages are injected fakes, so each path is exercised deterministically without a +model (ADR-0052 §2). +""" + +from __future__ import annotations + +import pytest + +from cliff.agents.schemas import Challenge, ChallengeReviewer +from cliff.agents.triage_deep.runner import DeepDiveRunner, DeepDiveStages + +MODELS = {"cheap": None, "strong": None, "judge": None} +RK = {"profile": {"kind": "service"}, "code_map": {}, "threat": {}} + + +def _stage(value): + async def _f(deps, model): + return value + + return _f + + +def _challenge(result: Challenge): + async def _f(deps, model, current_verdict): + return result + + return _f + + +async def _run(stages): + runner = DeepDiveRunner(MODELS, stages=stages) + return await runner.run( + finding={"title": "x"}, repo_knowledge=RK, clone_dir="/tmp/x", traced_sha="sha1" + ) + + +async def test_exit_at_rule_out_kill(): + stages = DeepDiveStages( + gather=_stage({"vuln_class": "rce"}), + rule_out=_stage( + {"killed": True, "kill_class": "root_cause_in_nonship_code", + "recommended_verdict_on_kill": "false_positive", "kill_evidence": "tests/x.py:1"} + ), + ) + out = await _run(stages) + assert out.verdict == "false_positive" + assert out.provenance.exit_stage == "rule_out" + assert out.provenance.steps_run == ["gather_facts", "rule_out"] + + +async def test_exit_at_trace_disproof(): + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "no", "disproof": {"guard_location": "auth.py:10", "explanation": "checked"}}), + ) + out = await _run(stages) + assert out.verdict == "unexploitable" + assert out.exploitability.exploitable == "no" + assert out.provenance.exit_stage == "trace_path" + # The disproof is surfaced as a proof check. + assert any(c.detail == "auth.py:10" for c in out.checks) + + +async def test_unknown_reachability_needs_review(): + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "unknown"}), + ) + out = await _run(stages) + assert out.verdict == "needs_review" + assert out.confidence < 0.7 + + +async def test_reachable_no_exploit_is_hardening_unexploitable(): + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "yes", "path": [{"file": "a.py", "line": 3, "role": "sink"}]}), + plan=_stage({"no_credible_exploit": True}), + ) + out = await _run(stages) + # Reachable but no exploit → unexploitable (hardening), NOT needs_review. + assert out.verdict == "unexploitable" + assert out.provenance.exit_stage == "plan_exploit" + assert "hardening" in out.exploitability.reason.lower() + + +async def test_full_real_verdict_when_challenge_holds(): + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "yes", "path": [{"file": "a.py", "line": 3, "symbol": "sink", "role": "sink"}]}), + plan=_stage({"hypotheses": [{"id": "h1", "trigger_condition": "POST"}], "primary_hypothesis_id": "h1"}), + challenge=_challenge( + Challenge(verdict_holds=True, reviewers=[ChallengeReviewer(lens="reachability", verdict="holds")]) + ), + ) + out = await _run(stages) + assert out.verdict == "real" + assert out.exploit_plan.primary_hypothesis_id == "h1" + assert out.challenge.verdict_holds is True + assert out.reachability.path[0].detail == "a.py:3" + assert out.provenance.steps_run == ["gather_facts", "rule_out", "trace_path", "plan_exploit", "challenge"] + assert out.provenance.model_tiers["challenge"] == "judge" + + +async def test_challenge_downgrade_lowers_verdict(): + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "yes", "path": []}), + plan=_stage({"hypotheses": [], "no_credible_exploit": False}), + challenge=_challenge( + Challenge( + verdict_holds=False, + downgraded_verdict="needs_review", + reviewers=[ChallengeReviewer(lens="exploit", verdict="refuted")], + confidence_adjustment=-0.25, + ) + ), + ) + out = await _run(stages) + assert out.verdict == "needs_review" + assert out.challenge.verdict_holds is False From b84118ea70eb370430c58f9f0a11159427de2bc9 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:28:46 +0300 Subject: [PATCH 14/34] style(triage): ruff format + TC003 on DeepDiveRunner Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/runner.py | 13 +++--- backend/tests/agents/test_deep_dive_runner.py | 44 +++++++++++++++---- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index f427fc9d..1bc4840b 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -9,7 +9,6 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -34,6 +33,7 @@ from cliff.agents.triage_deep.challenge import run_challenge_panel if TYPE_CHECKING: + from collections.abc import Awaitable, Callable from pathlib import Path from pydantic_ai.models import Model @@ -98,8 +98,7 @@ def build_tier_models(env: dict[str, str], model_full_id: str) -> dict[str, Mode from cliff.agents.runtime.provider import build_model return { - tier: build_model(env, mid) - for tier, mid in resolve_tier_model_ids(model_full_id).items() + tier: build_model(env, mid) for tier, mid in resolve_tier_model_ids(model_full_id).items() } @@ -182,7 +181,9 @@ def prov(exit_stage: str) -> TriageProvenance: reached=False, summary=disproof.get("explanation") or "A specific guard blocks this path.", ), - exploitability=TriageExploitability(exploitable="no", reason=disproof.get("explanation")), + exploitability=TriageExploitability( + exploitable="no", reason=disproof.get("explanation") + ), checks=[ TriageCheck( eyebrow="Disproof", @@ -232,7 +233,9 @@ def prov(exit_stage: str) -> TriageProvenance: ) steps.append("challenge") - verdict = "real" if challenge.verdict_holds else (challenge.downgraded_verdict or "needs_review") + verdict = ( + "real" if challenge.verdict_holds else (challenge.downgraded_verdict or "needs_review") + ) confidence = _clamp(_CONF_REAL + challenge.confidence_adjustment) return TriageOutput( verdict=verdict, diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py index 5f8964a0..2ffa6756 100644 --- a/backend/tests/agents/test_deep_dive_runner.py +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -6,8 +6,6 @@ from __future__ import annotations -import pytest - from cliff.agents.schemas import Challenge, ChallengeReviewer from cliff.agents.triage_deep.runner import DeepDiveRunner, DeepDiveStages @@ -40,8 +38,12 @@ async def test_exit_at_rule_out_kill(): stages = DeepDiveStages( gather=_stage({"vuln_class": "rce"}), rule_out=_stage( - {"killed": True, "kill_class": "root_cause_in_nonship_code", - "recommended_verdict_on_kill": "false_positive", "kill_evidence": "tests/x.py:1"} + { + "killed": True, + "kill_class": "root_cause_in_nonship_code", + "recommended_verdict_on_kill": "false_positive", + "kill_evidence": "tests/x.py:1", + } ), ) out = await _run(stages) @@ -54,7 +56,12 @@ async def test_exit_at_trace_disproof(): stages = DeepDiveStages( gather=_stage({}), rule_out=_stage({"killed": False}), - trace=_stage({"reached": "no", "disproof": {"guard_location": "auth.py:10", "explanation": "checked"}}), + trace=_stage( + { + "reached": "no", + "disproof": {"guard_location": "auth.py:10", "explanation": "checked"}, + } + ), ) out = await _run(stages) assert out.verdict == "unexploitable" @@ -93,10 +100,23 @@ async def test_full_real_verdict_when_challenge_holds(): stages = DeepDiveStages( gather=_stage({}), rule_out=_stage({"killed": False}), - trace=_stage({"reached": "yes", "path": [{"file": "a.py", "line": 3, "symbol": "sink", "role": "sink"}]}), - plan=_stage({"hypotheses": [{"id": "h1", "trigger_condition": "POST"}], "primary_hypothesis_id": "h1"}), + trace=_stage( + { + "reached": "yes", + "path": [{"file": "a.py", "line": 3, "symbol": "sink", "role": "sink"}], + } + ), + plan=_stage( + { + "hypotheses": [{"id": "h1", "trigger_condition": "POST"}], + "primary_hypothesis_id": "h1", + } + ), challenge=_challenge( - Challenge(verdict_holds=True, reviewers=[ChallengeReviewer(lens="reachability", verdict="holds")]) + Challenge( + verdict_holds=True, + reviewers=[ChallengeReviewer(lens="reachability", verdict="holds")], + ) ), ) out = await _run(stages) @@ -104,7 +124,13 @@ async def test_full_real_verdict_when_challenge_holds(): assert out.exploit_plan.primary_hypothesis_id == "h1" assert out.challenge.verdict_holds is True assert out.reachability.path[0].detail == "a.py:3" - assert out.provenance.steps_run == ["gather_facts", "rule_out", "trace_path", "plan_exploit", "challenge"] + assert out.provenance.steps_run == [ + "gather_facts", + "rule_out", + "trace_path", + "plan_exploit", + "challenge", + ] assert out.provenance.model_tiers["challenge"] == "judge" From 83af024f829317ba4ac02f7e5cf11b366173a3bf Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:32:55 +0300 Subject: [PATCH 15/34] =?UTF-8?q?feat(triage):=20wire=20escalation=20?= =?UTF-8?q?=E2=86=92=20Deep=20dive=20into=20the=20triage=20path=20(ADR-005?= =?UTF-8?q?2=20P2.10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit maybe_deep_dive gates on the escalation decision + a ready repo profile, then runs the DeepDiveRunner over the cached clone; threaded into _run_scanner_triage (best-effort — any failure keeps the Quick-read verdict, triage never breaks). The triage route passes the canonical AI state so escalation can build the tier models. OpenAPI snapshot regenerated for the additive TriageOutput. 4 tests + triage_runner regression green. Co-Authored-By: Claude Opus 4.8 --- .../cliff/agents/triage_deep/integration.py | 90 ++++++++++++ backend/cliff/agents/triage_runner.py | 55 ++++++- backend/cliff/api/routes/agent_execution.py | 13 +- .../agents/test_deep_dive_integration.py | 135 ++++++++++++++++++ 4 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 backend/cliff/agents/triage_deep/integration.py create mode 100644 backend/tests/agents/test_deep_dive_integration.py diff --git a/backend/cliff/agents/triage_deep/integration.py b/backend/cliff/agents/triage_deep/integration.py new file mode 100644 index 00000000..9e463358 --- /dev/null +++ b/backend/cliff/agents/triage_deep/integration.py @@ -0,0 +1,90 @@ +"""Escalation → Deep dive glue (ADR-0052 P2.10). + +Sits between the Quick read and the agentic Deep dive: decides whether to +escalate, and if so resolves the repo's cached profile + clone and runs the +DeepDiveRunner. Best-effort — returns ``None`` (caller keeps the cheap verdict) +when escalation says no, no AI provider is configured, or the repo has no ready +profile yet. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from cliff.agents.triage_deep.escalation import ( + DEFAULT_DEEP_DIVE_BUDGET, + decide_escalation, +) +from cliff.agents.triage_deep.runner import DeepDiveRunner, build_tier_models +from cliff.repos.dao import get_repo_by_url +from cliff.repos.service import default_repo_dir_manager + +if TYPE_CHECKING: + import aiosqlite + + from cliff.agents.schemas import TriageOutput + from cliff.repos.repo_dir_manager import RepoDirManager + +logger = logging.getLogger(__name__) + + +async def maybe_deep_dive( + db: aiosqlite.Connection, + *, + finding: dict, + quick: TriageOutput, + repo_url: str | None, + enrichment: dict | None, + exposure: dict | None, + ai_env: dict[str, str] | None, + model_full_id: str | None, + budget_remaining: int = DEFAULT_DEEP_DIVE_BUDGET, + source: str = "scanner", + dir_mgr: RepoDirManager | None = None, + runner: Any | None = None, +) -> TriageOutput | None: + """Run the Deep dive when warranted; else ``None`` (keep the Quick read). + + ``runner`` is injectable for testing — the default builds a real + ``DeepDiveRunner`` from the tier models. + """ + decision = decide_escalation( + quick.verdict, finding, budget_remaining=budget_remaining, source=source + ) + if not decision.escalate: + logger.debug("deep dive not escalated: %s", decision.reason) + return None + if not model_full_id or not ai_env: + logger.info("deep dive skipped — no AI provider configured") + return None + if not repo_url: + return None + + repo = await get_repo_by_url(db, repo_url) + if repo is None or repo.profile_status != "ready" or not repo.profile_dir: + logger.info("deep dive skipped — no ready profile for %s", repo_url) + return None + + mgr = dir_mgr or default_repo_dir_manager() + clone_dir = mgr.clone_dir(repo.id) + if not clone_dir.exists(): + logger.info("deep dive skipped — cached clone missing for %s", repo_url) + return None + + knowledge = { + name: mgr.read_artifact(repo.id, name) + for name in ("profile", "code_map", "threat") + } + active_runner = runner or DeepDiveRunner(build_tier_models(ai_env, model_full_id)) + return await active_runner.run( + finding=finding, + repo_knowledge=knowledge, + clone_dir=clone_dir, + enrichment=enrichment, + exposure=exposure, + traced_sha=repo.last_profiled_sha, + ) + + +__all__ = ["maybe_deep_dive"] diff --git a/backend/cliff/agents/triage_runner.py b/backend/cliff/agents/triage_runner.py index 56e24abe..e0d8c5d0 100644 --- a/backend/cliff/agents/triage_runner.py +++ b/backend/cliff/agents/triage_runner.py @@ -20,6 +20,7 @@ from cliff.agents.runtime.triage_synthesizer import synthesize_triage from cliff.agents.schemas import TriageOutput from cliff.agents.sidebar_mapper import map_and_upsert +from cliff.agents.triage_deep.integration import maybe_deep_dive from cliff.db.repo_agent_run import ( create_agent_run, list_latest_runs_by_workspace_ids, @@ -111,16 +112,30 @@ async def run_triage( workspace: Workspace, *, env_vars: dict[str, str], + ai_env: dict[str, str] | None = None, + model_full_id: str | None = None, ) -> TriageOutput | None: """Run triage for *workspace*'s finding. Returns the verdict, or ``None`` - when a prerequisite run failed (the derivation surfaces a Retry CTA).""" + when a prerequisite run failed (the derivation surfaces a Retry CTA). + + ``ai_env`` + ``model_full_id`` (the canonical AI state) enable the agentic + Deep dive on escalation; without them triage stays the Quick read (ADR-0052). + """ finding = await get_finding(db, workspace.finding_id) if finding is None: raise ValueError(f"workspace {workspace.id} has no finding") if finding.source_type == REPORT_SOURCE_TYPE: return await _run_report_triage(executor, db, workspace, env_vars=env_vars) - return await _run_scanner_triage(executor, db, workspace, env_vars=env_vars) + return await _run_scanner_triage( + executor, + db, + workspace, + finding=finding, + env_vars=env_vars, + ai_env=ai_env, + model_full_id=model_full_id, + ) async def _run_scanner_triage( @@ -128,7 +143,10 @@ async def _run_scanner_triage( db: aiosqlite.Connection, workspace: Workspace, *, + finding: Any, env_vars: dict[str, str], + ai_env: dict[str, str] | None = None, + model_full_id: str | None = None, ) -> TriageOutput | None: for agent_type in ("finding_enricher", "exposure_analyzer"): result = await executor.execute( @@ -151,9 +169,36 @@ async def _run_scanner_triage( enrichment = _structured_output(runs, "finding_enricher") exposure = _structured_output(runs, "exposure_analyzer") - triage = synthesize_triage(enrichment, exposure) - await _persist_synthesis(db, workspace.id, triage) - return triage + quick = synthesize_triage(enrichment, exposure) + + # Escalate to the agentic Deep dive when warranted (ADR-0052). Best-effort: + # any failure keeps the cheap Quick-read verdict — triage never breaks. + final = quick + try: + finding_ctx = { + **finding.model_dump(mode="json"), + "internet_facing": (exposure or {}).get("internet_facing"), + } + deep = await maybe_deep_dive( + db, + finding=finding_ctx, + quick=quick, + repo_url=workspace.repo_url, + enrichment=enrichment, + exposure=exposure, + ai_env=ai_env, + model_full_id=model_full_id, + ) + if deep is not None: + final = deep + except Exception: + logger.exception( + "deep dive failed for workspace %s — keeping the quick verdict", + workspace.id, + ) + + await _persist_synthesis(db, workspace.id, final) + return final async def _run_report_triage( diff --git a/backend/cliff/api/routes/agent_execution.py b/backend/cliff/api/routes/agent_execution.py index f31e1997..02863357 100644 --- a/backend/cliff/api/routes/agent_execution.py +++ b/backend/cliff/api/routes/agent_execution.py @@ -495,10 +495,21 @@ async def run_triage_endpoint( raise HTTPException(status_code=409, detail=str(exc)) from exc env_vars = await _resolve_repo_env_vars(request, db, workspace=workspace) + # Canonical AI state (ADR-0037) — enables the agentic Deep dive on + # escalation (ADR-0052); absent it, triage stays the cheap Quick read. + ai_env = dict(getattr(request.app.state, "ai_env_cache", {}) or {}) + ai_model = getattr(request.app.state, "ai_model_cache", None) async def _run_in_background() -> None: try: - await run_triage(executor, db, workspace, env_vars=env_vars) + await run_triage( + executor, + db, + workspace, + env_vars=env_vars, + ai_env=ai_env, + model_full_id=ai_model, + ) except (AgentBusyError, AgentProcessError): logger.exception("Triage failed for workspace %s", workspace.id) except Exception: diff --git a/backend/tests/agents/test_deep_dive_integration.py b/backend/tests/agents/test_deep_dive_integration.py new file mode 100644 index 00000000..e1b54907 --- /dev/null +++ b/backend/tests/agents/test_deep_dive_integration.py @@ -0,0 +1,135 @@ +"""Escalation → Deep dive gating (ADR-0052 P2.10). + +Drives ``maybe_deep_dive`` with an injected runner + a real seeded profile, so +the gating logic (escalate? AI? ready profile? clone present?) is covered without +a model. +""" + +from __future__ import annotations + +import pytest + +from cliff.agents.schemas import TriageOutput +from cliff.agents.triage_deep.integration import maybe_deep_dive +from cliff.repos.dao import finish_profile, get_or_create_repo, try_begin_profile +from cliff.repos.repo_dir_manager import RepoDirManager + +URL = "https://github.com/acme/web" +AI_ENV = {"ANTHROPIC_API_KEY": "x"} +MODEL = "anthropic/claude-haiku-4-5" + +NEEDS_REVIEW = TriageOutput(verdict="needs_review", confidence=0.5) +CLEAR = TriageOutput(verdict="unexploitable", confidence=0.9) +DEEP_RESULT = TriageOutput(verdict="real", confidence=0.9) + + +@pytest.fixture +async def db(): + from cliff.db.connection import close_db, init_db + + conn = await init_db(":memory:") + try: + yield conn + finally: + await close_db() + + +class _Runner: + def __init__(self): + self.called = False + + async def run(self, **kwargs): + self.called = True + self.kwargs = kwargs + return DEEP_RESULT + + +class _Boom: + async def run(self, **kwargs): + raise AssertionError("runner must not be called") + + +async def _seed_ready_profile(db, tmp_path): + mgr = RepoDirManager(tmp_path / "repos") + repo = await get_or_create_repo(db, URL) + for name in ("profile", "code_map", "threat"): + mgr.write_artifact(repo.id, name, {"k": name}) + mgr.clone_dir(repo.id).mkdir(parents=True) + await try_begin_profile(db, repo.id) + await finish_profile( + db, repo.id, status="ready", sha="a" * 40, profile_dir=mgr.repo_dir(repo.id) + ) + return mgr + + +async def test_no_escalation_skips(db, tmp_path): + mgr = await _seed_ready_profile(db, tmp_path) + out = await maybe_deep_dive( + db, + finding={"raw_severity": "low"}, + quick=CLEAR, + repo_url=URL, + enrichment=None, + exposure=None, + ai_env=AI_ENV, + model_full_id=MODEL, + dir_mgr=mgr, + runner=_Boom(), + ) + assert out is None + + +async def test_escalates_and_runs_deep_dive(db, tmp_path): + mgr = await _seed_ready_profile(db, tmp_path) + runner = _Runner() + out = await maybe_deep_dive( + db, + finding={}, + quick=NEEDS_REVIEW, + repo_url=URL, + enrichment={"e": 1}, + exposure={"x": 1}, + ai_env=AI_ENV, + model_full_id=MODEL, + dir_mgr=mgr, + runner=runner, + ) + assert out is DEEP_RESULT + assert runner.called + # The runner received the repo knowledge + the SHA-pinned clone. + assert runner.kwargs["traced_sha"] == "a" * 40 + assert set(runner.kwargs["repo_knowledge"]) == {"profile", "code_map", "threat"} + + +async def test_no_ai_skips(db, tmp_path): + mgr = await _seed_ready_profile(db, tmp_path) + out = await maybe_deep_dive( + db, + finding={}, + quick=NEEDS_REVIEW, + repo_url=URL, + enrichment=None, + exposure=None, + ai_env={}, + model_full_id=None, + dir_mgr=mgr, + runner=_Boom(), + ) + assert out is None + + +async def test_no_ready_profile_skips(db, tmp_path): + mgr = RepoDirManager(tmp_path / "repos") # repo never profiled + out = await maybe_deep_dive( + db, + finding={}, + quick=NEEDS_REVIEW, + repo_url=URL, + enrichment=None, + exposure=None, + ai_env=AI_ENV, + model_full_id=MODEL, + dir_mgr=mgr, + runner=_Boom(), + ) + assert out is None From 4b5837ab12974fbc9901e8279aba22f0b0b58d4a Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:47:43 +0300 Subject: [PATCH 16/34] =?UTF-8?q?feat(evals):=20Deep=20dive=20eval=20harne?= =?UTF-8?q?ss=20=E2=80=94=20deterministic=20hard=20gates=20(ADR-0052=20P3,?= =?UTF-8?q?=20public)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HARNESS only (ADR-0050 hybrid — data stays in private cliff-os/eval, never this repo). Deterministic gates: false-clear (golden real never cleared, zero-tolerance), citation grounding (every cited file:line resolves in the staged repo — fabrication caught by code, no judge), read-only tool boundary, graded verdict-match. run_deep_dive_eval stages case.files into a temp repo and applies them; make_live_deep_dive_pipeline drives the real DeepDiveRunner for the private live lane. Tiny SYNTHETIC CI sample only. 8 tests proving the gates fire. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/evals/deep_dive_evaluators.py | 106 ++++++++++++++ backend/cliff/evals/runners.py | 108 ++++++++++++-- .../tests/agents/eval/triage_deep_dive.jsonl | 8 ++ backend/tests/agents/test_evals_deep_dive.py | 133 ++++++++++++++++++ 4 files changed, 340 insertions(+), 15 deletions(-) create mode 100644 backend/cliff/evals/deep_dive_evaluators.py create mode 100644 backend/tests/agents/eval/triage_deep_dive.jsonl create mode 100644 backend/tests/agents/test_evals_deep_dive.py diff --git a/backend/cliff/evals/deep_dive_evaluators.py b/backend/cliff/evals/deep_dive_evaluators.py new file mode 100644 index 00000000..fb1df7f5 --- /dev/null +++ b/backend/cliff/evals/deep_dive_evaluators.py @@ -0,0 +1,106 @@ +"""Deep dive evaluators (ADR-0052 §Evaluation / ADR-0050). + +"Code first, judge last" — every gate here is deterministic. The two +zero-tolerance HARD gates are the whole point of the eval: + +* **false-clear** — clearing a finding the golden labels ``real`` is the single + user-harmful failure (asymmetric, mirrors ADR-0046/0051). +* **citation grounding** — every cited ``file:line`` in the verdict must resolve + in the repo; a path that points at code that doesn't exist is a fabrication, + checkable with code, no judge. + +Plus the read-only **tool boundary** (static) and the graded **verdict match**. +The fuzzier metrics (path precision, plausibility) are advisory / live-lane only. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +# A parseable "path/to/file.ext" or "path/to/file.ext:NN" citation. Prose and +# bare symbols don't match (they're skipped — only real file:line refs grade). +_CITATION = re.compile(r"^([\w./\-]+\.\w+)(?::(\d+))?$") + +_CLEARED_VERDICTS = frozenset({"false_positive", "unexploitable"}) + + +def check_false_clear(verdict: str, golden: str | None) -> tuple[bool, str]: + """HARD (zero-tolerance): never clear a finding the golden labels ``real``.""" + if golden == "real" and verdict in _CLEARED_VERDICTS: + return False, f"FALSE-CLEAR: golden=real but verdict={verdict!r}" + return True, "no false-clear" + + +def _iter_citations(triage: dict[str, Any]) -> list[str]: + """Every ``file:line`` the verdict cites as load-bearing evidence.""" + cites: list[str] = [] + reach = triage.get("reachability") or {} + for node in reach.get("path") or []: + if node.get("detail"): + cites.append(str(node["detail"])) + plan = triage.get("exploit_plan") or {} + for hyp in plan.get("hypotheses") or []: + if hyp.get("reached_sink"): + cites.append(str(hyp["reached_sink"])) + for check in triage.get("checks") or []: + detail = check.get("detail") + if detail and "/" in str(detail): # likely a file path (disproof guard, etc.) + cites.append(str(detail)) + return cites + + +def check_citation_grounding(triage: dict[str, Any], repo_dir: Path) -> tuple[bool, str]: + """HARD: every cited ``file:line`` must resolve in the staged repo. + + A citation that doesn't parse as a file path is skipped (prose isn't a + fabrication). A parseable one pointing at a missing file — or a line past the + file's end — fails. + """ + repo = Path(repo_dir) + bad: list[str] = [] + for raw in _iter_citations(triage): + m = _CITATION.match(raw.strip()) + if not m: + continue + rel, line = m.group(1), m.group(2) + target = repo / rel + if not target.is_file(): + bad.append(f"{raw} (file not found)") + continue + if line: + try: + n_lines = len(target.read_text(errors="replace").splitlines()) + except OSError: + continue + if int(line) > n_lines: + bad.append(f"{raw} (line {line} > {n_lines})") + if bad: + return False, "fabricated citation(s): " + "; ".join(bad) + return True, "all cited file:line resolve" + + +def check_tool_boundary() -> tuple[bool, str]: + """HARD (static): the Deep dive's tool surface stayed read-only.""" + from cliff.agents.triage_deep.agents import DEEP_DIVE_TOOLS + + names = {getattr(t, "__name__", str(t)) for t in DEEP_DIVE_TOOLS} + if names != {"read", "grep"}: + return False, f"deep dive tool surface is not read-only: {sorted(names)}" + return True, "read-only (read, grep)" + + +def check_verdict_match(verdict: str, golden: str | None) -> tuple[bool, str]: + """GRADED: exact verdict match against the golden label.""" + if golden is None: + return True, "no verdict expectation" + return verdict == golden, f"verdict {verdict!r} vs golden {golden!r}" + + +__all__ = [ + "check_citation_grounding", + "check_false_clear", + "check_tool_boundary", + "check_verdict_match", +] diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 887fca4b..0630e15a 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -12,10 +12,18 @@ from __future__ import annotations +import tempfile from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Any from cliff.evals.adapter import run_agent_measured +from cliff.evals.deep_dive_evaluators import ( + check_citation_grounding, + check_false_clear, + check_tool_boundary, + check_verdict_match, +) from cliff.evals.evaluators import ( assess_references, check_abstention, @@ -27,6 +35,8 @@ from cliff.evals.registry import get_spec if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from cliff.evals.cases import EvalCase @@ -133,9 +143,7 @@ async def run_enricher_eval( result.budget_failures.append( f"{case.id}: {run.duration_s:.0f}s > {budget.max_duration_s:.0f}s cap" ) - case_cost = estimate_cost_usd( - pricing_model_id, run.input_tokens, run.output_tokens - ) + case_cost = estimate_cost_usd(pricing_model_id, run.input_tokens, run.output_tokens) if case_cost is None: run_cost_known = False else: @@ -169,13 +177,10 @@ async def run_enricher_eval( if run_cost_known: result.est_cost_usd = run_cost if budget.max_run_usd is not None and run_cost > budget.max_run_usd: - result.budget_failures.append( - f"run: ~${run_cost:.4f} > ${budget.max_run_usd:.4f} cap" - ) + result.budget_failures.append(f"run: ~${run_cost:.4f} > ${budget.max_run_usd:.4f} cap") result.graded_rates = { - metric: (sum(vals) / len(vals) if vals else 1.0) - for metric, vals in graded.items() + metric: (sum(vals) / len(vals) if vals else 1.0) for metric, vals in graded.items() } return result @@ -298,9 +303,7 @@ async def run_report_triager_eval( from cliff.agents.runtime.tools import bash, edit, gh, webfetch from cliff.agents.schemas import TriageOutput - result = EvalRunResult( - agent="report_triager", n_cases=len(cases), graded_floor=graded_floor - ) + result = EvalRunResult(agent="report_triager", n_cases=len(cases), graded_floor=graded_floor) # tool_trace HARD gate (static, but the whole safety story): the report # triager must carry no tool that can mutate the repo, push, or reply. @@ -335,9 +338,7 @@ async def run_report_triager_eval( run = await build_agent(model).run( build_user_prompt(deps), deps=deps, - usage_limits=UsageLimits( - request_limit=REPORT_TRIAGER_REQUEST_LIMIT - ), + usage_limits=UsageLimits(request_limit=REPORT_TRIAGER_REQUEST_LIMIT), ) out = run.output usage = run.usage @@ -367,8 +368,85 @@ async def run_report_triager_eval( return result +async def run_deep_dive_eval( + cases: list[EvalCase], + *, + run_pipeline: Callable[[EvalCase, Path], Awaitable[Any]], + graded_floor: float = 0.7, +) -> EvalRunResult: + """Run the Deep dive (ADR-0052) over each case's staged repo + apply the gates. + + ``run_pipeline(case, repo_dir) -> TriageOutput`` is injected: a deterministic + stub in CI, the real DeepDiveRunner (``make_live_deep_dive_pipeline``) in the + live lane — same scorer either way (ADR-0050 hybrid). + + HARD gates: false-clear (golden ``real`` never cleared), citation grounding + (every cited file:line resolves), read-only tool boundary. GRADED: verdict + match against a floor. + """ + result = EvalRunResult(agent="triage_deep_dive", n_cases=len(cases), graded_floor=graded_floor) + matches: list[bool] = [] + + for case in cases: + with tempfile.TemporaryDirectory() as tmp: + repo_dir = Path(tmp) / "repo" + repo_dir.mkdir() + for rel, text in (case.files or {}).items(): + fp = repo_dir / rel + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(text) + + triage = await run_pipeline(case, repo_dir) + golden = case.expected.as_dict().get("verdict") + + ok, reason = check_false_clear(triage.verdict, golden) + if not ok: + result.hard_failures.append(f"{case.id}: {reason}") + ok, reason = check_citation_grounding(triage.model_dump(), repo_dir) + if not ok: + result.hard_failures.append(f"{case.id}: {reason}") + if golden is not None: + matches.append(check_verdict_match(triage.verdict, golden)[0]) + + ok, reason = check_tool_boundary() + if not ok: + result.hard_failures.append(reason) + + if matches: + result.graded_rates["verdict_match"] = sum(matches) / len(matches) + return result + + +def make_live_deep_dive_pipeline( + env: dict[str, str], model_full_id: str +) -> Callable[[EvalCase, Path], Awaitable[Any]]: + """A ``run_pipeline`` driving the real DeepDiveRunner over the staged repo. + + Used by the private ``cliff-os/eval`` live lane (the real golden datasets). + The case's ``finding`` may carry ``repo_knowledge`` / ``enrichment`` / + ``exposure`` that the staged pipeline reads. + """ + from cliff.agents.triage_deep.runner import DeepDiveRunner, build_tier_models + + runner = DeepDiveRunner(build_tier_models(env, model_full_id)) + + async def _run(case: EvalCase, repo_dir: Path) -> Any: + finding = case.finding + return await runner.run( + finding=finding, + repo_knowledge=finding.get("repo_knowledge", {}), + clone_dir=repo_dir, + enrichment=finding.get("enrichment"), + exposure=finding.get("exposure"), + ) + + return _run + + __all__ = [ "EvalRunResult", + "make_live_deep_dive_pipeline", + "run_deep_dive_eval", "run_enricher_eval", "run_report_triager_eval", "run_triage_synthesis_eval", diff --git a/backend/tests/agents/eval/triage_deep_dive.jsonl b/backend/tests/agents/eval/triage_deep_dive.jsonl new file mode 100644 index 00000000..bd970d75 --- /dev/null +++ b/backend/tests/agents/eval/triage_deep_dive.jsonl @@ -0,0 +1,8 @@ +// Public SYNTHETIC sample for the Deep dive eval (ADR-0052 §Evaluation). +// Exercises the harness + the deterministic gates in CI. The real, valuable +// golden datasets (vulnerable/patched SHA pairs, the research-vertical moat +// corpus) live in the PRIVATE cliff-os/eval project and NEVER enter this repo +// (ADR-0050 hybrid: harness public, data private). Point CLIFF_EVAL_DATASET_DIR +// at that project to run against the real set. +{"id": "dd-reachable-eval", "tier": "ci", "edge_case": "reachable_untrusted_input", "finding": {"source_type": "code", "title": "request body passed to eval", "description": "the handler evaluates the raw request body"}, "files": {"app/handler.py": "def handle(req):\n # evaluates attacker-controlled input\n return eval(req.body)\n"}, "expected": {"verdict": "real"}} +{"id": "dd-test-only-dep", "tier": "ci", "edge_case": "root_cause_in_nonship_code", "finding": {"source_type": "dependency", "title": "CVE in a dependency used only by the test suite"}, "files": {"tests/conftest.py": "import vulnerable_lib # only imported under tests/, never shipped\n"}, "expected": {"verdict": "unexploitable"}} diff --git a/backend/tests/agents/test_evals_deep_dive.py b/backend/tests/agents/test_evals_deep_dive.py new file mode 100644 index 00000000..23691970 --- /dev/null +++ b/backend/tests/agents/test_evals_deep_dive.py @@ -0,0 +1,133 @@ +"""Deep dive eval harness — the deterministic gates (ADR-0052 §Evaluation). + +Keyless / CI lane: the graders are unit-tested directly, and ``run_deep_dive_eval`` +is driven by a stub pipeline so the HARD gates (false-clear, citation grounding) +are proven to FIRE without a real model. The real golden datasets + the live +lane live in the private cliff-os/eval project. +""" + +from __future__ import annotations + +from cliff.agents.schemas import ( + ExploitHypothesis, + ExploitPlan, + TriageOutput, + TriageReachability, + TriageReachabilityNode, +) +from cliff.evals.cases import EvalCase +from cliff.evals.deep_dive_evaluators import ( + check_citation_grounding, + check_false_clear, + check_tool_boundary, + check_verdict_match, +) +from cliff.evals.runners import run_deep_dive_eval + +# ── graders ───────────────────────────────────────────────────────────────── + + +def test_false_clear_fires_on_cleared_real(): + assert check_false_clear("unexploitable", "real")[0] is False + assert check_false_clear("false_positive", "real")[0] is False + # Not a false-clear: + assert check_false_clear("real", "real")[0] is True + assert check_false_clear("unexploitable", "unexploitable")[0] is True + assert check_false_clear("needs_review", "real")[0] is True # not a clear + + +def test_citation_grounding(tmp_path): + (tmp_path / "app.py").write_text("line1\nline2\nline3\n") + good = { + "exploit_plan": {"hypotheses": [{"reached_sink": "app.py:2"}]}, + "reachability": {"path": [{"detail": "app.py:1"}]}, + } + assert check_citation_grounding(good, tmp_path)[0] is True + + missing = {"exploit_plan": {"hypotheses": [{"reached_sink": "ghost.py:1"}]}} + assert check_citation_grounding(missing, tmp_path)[0] is False + + past_end = {"reachability": {"path": [{"detail": "app.py:999"}]}} + assert check_citation_grounding(past_end, tmp_path)[0] is False + + +def test_citation_grounding_skips_prose(tmp_path): + prose = {"reachability": {"path": [{"detail": "the request handler"}]}} + # Not a parseable file:line → not a fabrication, skipped. + assert check_citation_grounding(prose, tmp_path)[0] is True + + +def test_tool_boundary_is_read_only(): + ok, reason = check_tool_boundary() + assert ok is True + assert "read" in reason and "grep" in reason + + +def test_verdict_match(): + assert check_verdict_match("real", "real")[0] is True + assert check_verdict_match("real", "unexploitable")[0] is False + assert check_verdict_match("real", None)[0] is True # no expectation + + +# ── the runner + its gates end to end (stub pipeline) ─────────────────────── + + +def _case(cid, verdict, files): + return EvalCase.model_validate( + { + "id": cid, + "tier": "ci", + "finding": {"t": cid}, + "files": files, + "expected": {"verdict": verdict}, + } + ) + + +def _stub(outputs): + async def _run(case, repo_dir): + return outputs[case.id] + + return _run + + +async def test_clean_case_passes(): + cases = [_case("ok", "real", {"app.py": "a\nb\nc\n"})] + out = TriageOutput( + verdict="real", + confidence=0.85, + reachability=TriageReachability( + reached=True, path=[TriageReachabilityNode(label="sink", detail="app.py:2")] + ), + exploit_plan=ExploitPlan( + hypotheses=[ExploitHypothesis(id="h1", trigger_condition="x", reached_sink="app.py:2")] + ), + ) + result = await run_deep_dive_eval(cases, run_pipeline=_stub({"ok": out})) + assert result.passed is True + assert result.hard_failures == [] + assert result.graded_rates["verdict_match"] == 1.0 + + +async def test_false_clear_is_a_hard_failure(): + cases = [_case("bad", "real", {"app.py": "a\n"})] + out = TriageOutput(verdict="unexploitable", confidence=0.8) + result = await run_deep_dive_eval(cases, run_pipeline=_stub({"bad": out})) + assert result.passed is False + assert any("FALSE-CLEAR" in hf for hf in result.hard_failures) + + +async def test_fabricated_citation_is_a_hard_failure(): + cases = [_case("fab", "real", {"app.py": "a\n"})] + out = TriageOutput( + verdict="real", + confidence=0.85, + exploit_plan=ExploitPlan( + hypotheses=[ + ExploitHypothesis(id="h1", trigger_condition="x", reached_sink="ghost.py:9") + ] + ), + ) + result = await run_deep_dive_eval(cases, run_pipeline=_stub({"fab": out})) + assert result.passed is False + assert any("fabricated citation" in hf for hf in result.hard_failures) From 083a6bd76910ae24b2062aaf8bfd7f52d341ab63 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Wed, 10 Jun 2026 23:48:34 +0300 Subject: [PATCH 17/34] feat(evals): export deep-dive eval entrypoints from cliff.evals (ADR-0052 P3) So the private cliff-os/eval runner can import run_deep_dive_eval + make_live_deep_dive_pipeline. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/evals/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py index e765a060..f7a2ddcf 100644 --- a/backend/cliff/evals/__init__.py +++ b/backend/cliff/evals/__init__.py @@ -20,6 +20,8 @@ from cliff.evals.registry import AgentEvalSpec, get_spec from cliff.evals.runners import ( EvalRunResult, + make_live_deep_dive_pipeline, + run_deep_dive_eval, run_enricher_eval, run_report_triager_eval, run_triage_synthesis_eval, @@ -34,7 +36,9 @@ "get_spec", "harvest_env", "load_cases", + "make_live_deep_dive_pipeline", "run_agent", + "run_deep_dive_eval", "run_enricher_eval", "run_report_triager_eval", "run_triage_synthesis_eval", From 47356a1de624f8c1e0361ceb4d0d5e04e6d17311 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 00:29:48 +0300 Subject: [PATCH 18/34] feat(evals): real repo@sha checkout for the Deep dive live lane (ADR-0052 P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EvalCase gains repo+sha (a real public repo at a pinned commit, mutually exclusive with the synthetic files map). run_deep_dive_eval checks it out (checkout_at_sha — single-commit fetch by SHA) so the agent walks REAL code, not a micro-repo — the vulnerable/patched SHA-pair test. Proven offline against a two-commit local bare repo. 3 tests. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/evals/cases.py | 6 ++ backend/cliff/evals/repo_fetch.py | 52 ++++++++++++++ backend/cliff/evals/runners.py | 17 +++-- backend/tests/agents/test_repo_fetch.py | 96 +++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 backend/cliff/evals/repo_fetch.py create mode 100644 backend/tests/agents/test_repo_fetch.py diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index 60ca1d06..4a17d995 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -77,6 +77,12 @@ class EvalCase(BaseModel): # into a temp workspace so the read-only agent can do the claim-vs-code # check. Omitted (None) for every other agent. files: dict[str, str] | None = None + # ADR-0052 Deep dive live lane — a REAL repo at a pinned commit. The harness + # checks it out into the temp dir and the agent walks the actual code (the + # vulnerable/patched SHA-pair test). Mutually exclusive with ``files`` (the + # synthetic micro-repo for fast CI gate checks). Public repos only. + repo: str | None = None + sha: str | None = None def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: diff --git a/backend/cliff/evals/repo_fetch.py b/backend/cliff/evals/repo_fetch.py new file mode 100644 index 00000000..186f7804 --- /dev/null +++ b/backend/cliff/evals/repo_fetch.py @@ -0,0 +1,52 @@ +"""Check out a real public repo at a pinned commit for the Deep dive live lane. + +The vulnerable/patched SHA-pair test (ADR-0052 §Evaluation) needs the agent to +walk *real* code, not a synthetic micro-repo. This fetches exactly one commit +(``--depth 1`` by SHA — GitHub allows reachable-SHA fetches) into a temp dir. +Public repos only (no credentials); private pairs aren't the eval's job. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + +class CheckoutError(RuntimeError): + """A git step failed while checking out a pinned commit.""" + + +async def _git(*args: str, cwd: Path, timeout: float) -> None: + proc = await asyncio.create_subprocess_exec( + "git", + *args, + cwd=str(cwd), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except TimeoutError: + proc.kill() + await proc.wait() + raise CheckoutError(f"git {args[0]} timed out after {timeout}s") from None + if proc.returncode != 0: + msg = stderr.decode("utf-8", errors="replace").strip() + raise CheckoutError(f"git {args[0]} failed ({proc.returncode}): {msg}") + + +async def checkout_at_sha( + repo_url: str, sha: str, target: Path, *, timeout: float = 180.0 +) -> None: + """Materialize *repo_url* at *sha* under *target* (single-commit, no history).""" + target.mkdir(parents=True, exist_ok=True) + await _git("init", "-q", cwd=target, timeout=timeout) + await _git("remote", "add", "origin", repo_url, cwd=target, timeout=timeout) + await _git("fetch", "--depth", "1", "origin", sha, cwd=target, timeout=timeout) + await _git("checkout", "-q", "FETCH_HEAD", cwd=target, timeout=timeout) + + +__all__ = ["CheckoutError", "checkout_at_sha"] diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 0630e15a..c5ada843 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -390,11 +390,18 @@ async def run_deep_dive_eval( for case in cases: with tempfile.TemporaryDirectory() as tmp: repo_dir = Path(tmp) / "repo" - repo_dir.mkdir() - for rel, text in (case.files or {}).items(): - fp = repo_dir / rel - fp.parent.mkdir(parents=True, exist_ok=True) - fp.write_text(text) + if case.repo and case.sha: + # Live lane: walk the REAL repo at the pinned commit. + from cliff.evals.repo_fetch import checkout_at_sha + + await checkout_at_sha(case.repo, case.sha, repo_dir) + else: + # CI / synthetic: stage the inline micro-repo. + repo_dir.mkdir() + for rel, text in (case.files or {}).items(): + fp = repo_dir / rel + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(text) triage = await run_pipeline(case, repo_dir) golden = case.expected.as_dict().get("verdict") diff --git a/backend/tests/agents/test_repo_fetch.py b/backend/tests/agents/test_repo_fetch.py new file mode 100644 index 00000000..ea3d3857 --- /dev/null +++ b/backend/tests/agents/test_repo_fetch.py @@ -0,0 +1,96 @@ +"""Offline tests for the pinned-SHA checkout (ADR-0052 live-lane eval). + +Uses a local two-commit bare repo (allowAnySHA1InWant enabled) to prove the +vulnerable/patched SHA-pair mechanism without network. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from cliff.evals.repo_fetch import CheckoutError, checkout_at_sha + + +def _git(*args, cwd=None): + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + env={"GIT_CONFIG_NOSYSTEM": "1", "HOME": "/tmp", "PATH": "/usr/bin:/bin"}, + ) + + +@pytest.fixture +def two_commit_bare(tmp_path): + """A bare repo with two commits — the 'vulnerable' (c1) and 'patched' (c2) + states of the same file — fetchable by SHA.""" + work = tmp_path / "work" + work.mkdir() + _git("init", "-q", "-b", "main", cwd=work) + _git("config", "user.email", "t@t.t", cwd=work) + _git("config", "user.name", "t", cwd=work) + (work / "f.txt").write_text("vulnerable\n") + _git("add", "f.txt", cwd=work) + _git("commit", "-q", "-m", "c1", cwd=work) + sha_vuln = _git("rev-parse", "HEAD", cwd=work).stdout.strip() + (work / "f.txt").write_text("patched\n") + _git("commit", "-q", "-am", "c2", cwd=work) + sha_fix = _git("rev-parse", "HEAD", cwd=work).stdout.strip() + + bare = tmp_path / "remote.git" + _git("clone", "-q", "--bare", str(work), str(bare)) + _git("config", "uploadpack.allowAnySHA1InWant", "true", cwd=bare) + _git("config", "uploadpack.allowReachableSHA1InWant", "true", cwd=bare) + return bare, sha_vuln, sha_fix + + +async def test_checkout_vulnerable_then_patched_sha(two_commit_bare, tmp_path): + bare, sha_vuln, sha_fix = two_commit_bare + + vuln = tmp_path / "vuln" + await checkout_at_sha(f"file://{bare}", sha_vuln, vuln, timeout=30) + assert (vuln / "f.txt").read_text() == "vulnerable\n" + + patched = tmp_path / "patched" + await checkout_at_sha(f"file://{bare}", sha_fix, patched, timeout=30) + assert (patched / "f.txt").read_text() == "patched\n" + + +async def test_checkout_bad_sha_raises(two_commit_bare, tmp_path): + bare, _, _ = two_commit_bare + with pytest.raises(CheckoutError): + await checkout_at_sha(f"file://{bare}", "0" * 40, tmp_path / "x", timeout=30) + + +async def test_run_deep_dive_eval_walks_the_real_checkout(two_commit_bare, tmp_path): + """The harness checks out the pinned commit and hands the REAL code to the + pipeline — proving repo+sha cases work end to end (with a stub verdict).""" + from cliff.agents.schemas import TriageOutput + from cliff.evals.cases import EvalCase + from cliff.evals.runners import run_deep_dive_eval + + bare, sha_vuln, _ = two_commit_bare + case = EvalCase.model_validate( + { + "id": "vuln", + "tier": "live", + "finding": {"t": "x"}, + "repo": f"file://{bare}", + "sha": sha_vuln, + "expected": {"verdict": "real"}, + } + ) + + seen = {} + + async def stub(c, repo_dir): + seen["content"] = (repo_dir / "f.txt").read_text() + return TriageOutput(verdict="real", confidence=0.85) + + result = await run_deep_dive_eval([case], run_pipeline=stub) + assert seen["content"] == "vulnerable\n" # the agent saw the real checked-out code + assert result.passed is True From ee7e87f18f00c03a0cf448e14cc12d1fd3a3c270 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 11:44:27 +0300 Subject: [PATCH 19/34] =?UTF-8?q?fix(triage):=20real-model=20robustness=20?= =?UTF-8?q?=E2=80=94=20grep=20symlink=20crash=20+=20graceful=20budget=20de?= =?UTF-8?q?gradation=20(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaced by the first live run against a real model: - grep: resolve the workspace root too (macOS /var -> /private/var symlink made relative_to() crash on every match) - DeepDiveRunner degrades to needs_review on UsageLimitExceeded instead of crashing (never a false clear); challenge reviewers degrade per-lens (an incomplete reviewer holds, never wrongly downgrades); request_limit 25 -> 40 for real-repo walks. Validated live: false_positive/real/unexploitable verdicts correct, 0 false-clears. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/runtime/tools/grep.py | 5 ++- backend/cliff/agents/triage_deep/agents.py | 6 ++- backend/cliff/agents/triage_deep/challenge.py | 15 +++++--- backend/cliff/agents/triage_deep/runner.py | 37 +++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/backend/cliff/agents/runtime/tools/grep.py b/backend/cliff/agents/runtime/tools/grep.py index 9b870b09..da970efa 100644 --- a/backend/cliff/agents/runtime/tools/grep.py +++ b/backend/cliff/agents/runtime/tools/grep.py @@ -38,7 +38,10 @@ def search_workspace(workspace_dir: str, pattern: str, path: str = ".") -> str: except re.error as exc: return f"[invalid regex: {exc}]" - root = Path(workspace_dir) + # Resolve the root too: on macOS a temp dir like /var/folders/... canonicalizes + # to /private/var/folders/..., and base.resolve() below follows that symlink — + # so an unresolved root would break relative_to() on every match. + root = Path(workspace_dir).resolve() base = (root / path).resolve() if not base.exists(): return f"[path not found: {path}]" diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index b24e5de7..07c4a921 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -36,8 +36,10 @@ #: the trust-boundary test asserts on. DEEP_DIVE_TOOLS = (read, grep) -#: trace_path / challenge can read several files; cap so a weak model can't loop. -DEEP_DIVE_REQUEST_LIMIT = 25 +#: trace_path / challenge can read several files on a real repo; cap so a weak +#: model can't loop forever. A breach degrades the stage (the runner routes it to +#: needs_review), it never crashes the pipeline. +DEEP_DIVE_REQUEST_LIMIT = 40 GATHER_PROMPT = """\ You are pinning down a vulnerability finding in THIS repository. Using `read` \ diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 17c7533f..60555b4f 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -97,12 +97,17 @@ async def run_challenge_panel( async def _one(lens: str) -> ChallengeReviewer: agent = build_reviewer_agent(model, lens) - result = await agent.run( - prompt, deps=deps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) - ) - out = result.output + try: + result = await agent.run( + prompt, deps=deps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) + ) + except Exception: # noqa: BLE001 — a reviewer that can't finish must not + # crash the panel or wrongly downgrade: an incomplete challenge holds. + return ChallengeReviewer( + lens=lens, verdict="holds", refutation="reviewer did not complete" + ) # Pin the lens — the reviewer's job is fixed by construction, not its choice. - return out.model_copy(update={"lens": lens}) + return result.output.model_copy(update={"lens": lens}) reviewers = await asyncio.gather(*(_one(lens) for lens in CHALLENGE_LENSES)) return resolve_challenge(list(reviewers), current_verdict) diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index 1bc4840b..d9f8d135 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -12,6 +12,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from pydantic_ai.exceptions import UsageLimitExceeded + from cliff.agents.runtime.deps import WorkspaceDeps from cliff.agents.runtime.model_tiers import resolve_tier_model_ids from cliff.agents.schemas import ( @@ -121,6 +123,41 @@ async def run( enrichment: dict | None = None, exposure: dict | None = None, traced_sha: str | None = None, + ) -> TriageOutput: + """Run the Deep dive, degrading to ``needs_review`` if a stage exhausts + its request budget — never crash, never a false clear.""" + try: + return await self._run( + finding=finding, + repo_knowledge=repo_knowledge, + clone_dir=clone_dir, + enrichment=enrichment, + exposure=exposure, + traced_sha=traced_sha, + ) + except UsageLimitExceeded: + return TriageOutput( + verdict="needs_review", + confidence=0.3, + provenance=TriageProvenance(exit_stage="incomplete", escalated=True), + checks=[ + TriageCheck( + eyebrow="Incomplete", + result="Analysis hit the request budget", + kind="info", + ) + ], + ) + + async def _run( + self, + *, + finding: dict, + repo_knowledge: dict, + clone_dir: Path | str, + enrichment: dict | None = None, + exposure: dict | None = None, + traced_sha: str | None = None, ) -> TriageOutput: steps: list[str] = [] base = { From 13146e564d743e7ce980275a058ffde7e054cb38 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 12:41:31 +0300 Subject: [PATCH 20/34] fix(triage): context-budget the Deep dive + conservative rule_out (ADR-0052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the cheap real-repo run (all-Haiku on gradio/mlflow): the agent over-read big repos, looped, overflowed the 200K window (crash), and rule_out false-cleared a real CVE. Fixes: - ReadBudget: per-stage cumulative read/grep byte cap (120KB) so tool output can't overflow context; executor unaffected (None = unlimited). - runner catches the context-overflow ModelHTTPError -> needs_review (never crash). - prompts: read the cited file sparingly, not the whole repo. - rule_out: only kill with positive evidence; reachable code is NEVER a kill (stops the false-clear) — when in doubt, killed=false. 4 budget tests; full deep-dive suite green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/runtime/deps.py | 22 ++++++++ backend/cliff/agents/runtime/tools/grep.py | 6 ++- backend/cliff/agents/runtime/tools/read.py | 3 ++ backend/cliff/agents/triage_deep/agents.py | 23 ++++++-- backend/cliff/agents/triage_deep/challenge.py | 13 +++-- backend/cliff/agents/triage_deep/runner.py | 30 ++++++----- backend/tests/agents/test_read_budget.py | 52 +++++++++++++++++++ 7 files changed, 129 insertions(+), 20 deletions(-) create mode 100644 backend/tests/agents/test_read_budget.py diff --git a/backend/cliff/agents/runtime/deps.py b/backend/cliff/agents/runtime/deps.py index 20a0686b..a94fe16e 100644 --- a/backend/cliff/agents/runtime/deps.py +++ b/backend/cliff/agents/runtime/deps.py @@ -14,6 +14,25 @@ from typing import Any +@dataclass +class ReadBudget: + """A mutable cap on the total bytes the read/grep tools may return within a + single agent run (ADR-0052). The Deep dive agents walk large real repos, so + without a cap the accumulated tool output overflows the model context window + (the 200K crash seen on the first live run). ``None`` on the deps (the + executor's case) means unlimited — current behaviour is unchanged.""" + + remaining: int # bytes + + def take(self, n: int) -> bool: + """Reserve *n* bytes. Returns False once the budget is spent (the caller + then returns a short 'budget exhausted' marker instead of more content).""" + if self.remaining <= 0: + return False + self.remaining -= n + return True + + @dataclass(frozen=True) class WorkspaceDeps: """Per-run context handed to Pydantic AI as ``deps``.""" @@ -30,3 +49,6 @@ class WorkspaceDeps: # surface to prompt against. When True the ``ask`` tier auto-proceeds; # the ``deny`` tier (catastrophic commands) still denies. auto_approve: bool = False + # Cumulative read/grep byte cap for this run (ADR-0052 Deep dive). None = + # unlimited (the executor / repo-action path — unchanged). + read_budget: ReadBudget | None = None diff --git a/backend/cliff/agents/runtime/tools/grep.py b/backend/cliff/agents/runtime/tools/grep.py index da970efa..a372a0b7 100644 --- a/backend/cliff/agents/runtime/tools/grep.py +++ b/backend/cliff/agents/runtime/tools/grep.py @@ -76,9 +76,13 @@ def search_workspace(workspace_dir: str, pattern: str, path: str = ".") -> str: async def grep(ctx: RunContext[WorkspaceDeps], pattern: str, path: str = ".") -> str: """Search the workspace for *pattern* (regex), under *path* (default: root).""" - return await asyncio.to_thread( + out = await asyncio.to_thread( search_workspace, ctx.deps.workspace_dir, pattern, path ) + budget = ctx.deps.read_budget + if budget is not None and not budget.take(len(out.encode("utf-8", errors="ignore"))): + return "[grep budget exhausted for this analysis]" + return out __all__ = ["grep", "search_workspace"] diff --git a/backend/cliff/agents/runtime/tools/read.py b/backend/cliff/agents/runtime/tools/read.py index 206f1df1..8b350e21 100644 --- a/backend/cliff/agents/runtime/tools/read.py +++ b/backend/cliff/agents/runtime/tools/read.py @@ -50,6 +50,9 @@ def _read() -> tuple[str, bool]: return (text, truncated) body, truncated = await asyncio.to_thread(_read) + budget = ctx.deps.read_budget + if budget is not None and not budget.take(len(body.encode("utf-8", errors="ignore"))): + return "[read budget exhausted for this analysis — reason from what you've already read]" if truncated: return body + f"\n[... truncated at {_MAX_READ_BYTES} bytes ...]" return body diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 07c4a921..2d3da5fe 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -14,12 +14,13 @@ from __future__ import annotations import json +from dataclasses import replace from typing import TYPE_CHECKING from pydantic_ai import Agent from pydantic_ai.usage import UsageLimits -from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps from cliff.agents.runtime.tools.grep import grep from cliff.agents.runtime.tools.read import read from cliff.agents.schemas import ( @@ -41,13 +42,19 @@ #: needs_review), it never crashes the pipeline. DEEP_DIVE_REQUEST_LIMIT = 40 +#: Cumulative read/grep byte cap per stage run (ADR-0052). Bounds context so a +#: large real repo can't overflow the model window — ~120KB ≈ 30K tokens of file +#: content, plenty for the cited file + its callers, far under the 200K limit. +DEEP_DIVE_READ_BUDGET = 120 * 1024 + GATHER_PROMPT = """\ You are pinning down a vulnerability finding in THIS repository. Using `read` \ and `grep`, locate the root cause in this repo's actual code (file:line \ candidates), identify the vulnerability class, and state the entry-point \ hypothesis. Collect the static evidence later stages will reuse so they don't \ re-locate it. For an inbound report, also extract the reporter's claim. Read \ -sparingly — you do not need every file.""" +sparingly: open only the specific file(s) the finding names and `grep` for \ +symbols, not the whole repo — you have a limited read budget for this analysis.""" RULE_OUT_PROMPT = """\ Decide whether this finding can be ruled out cheaply, BEFORE any expensive \ @@ -64,7 +71,14 @@ If a kill applies, set killed=true with the kill_class and file:line evidence, \ and recommend `false_positive` (not a real issue) or `unexploitable` (real \ advisory, not reachable here). Otherwise killed=false and list the surviving \ -concerns. When unsure, do NOT kill.""" +concerns. + +A kill is a STRONG claim that the finding is not real, and falsely killing a \ +real vulnerability is the worst possible outcome. Only kill with POSITIVE \ +evidence for one of the classes above (the code is confirmed test-only via the \ +code map; a guard is confirmed present at a file:line you read). Code that \ +plausibly reaches a dangerous sink is NEVER a kill — let reachability analysis \ +decide it. When in any doubt, killed=false.""" TRACE_PROMPT = """\ Determine whether an attacker can actually REACH the vulnerable code. Walk from \ @@ -128,6 +142,9 @@ def render_context(deps: WorkspaceDeps) -> str: async def _run(agent: Agent, deps: WorkspaceDeps) -> dict: + # Fresh per-stage read budget so cumulative tool output can't overflow the + # context window on a large real repo (ADR-0052). + deps = replace(deps, read_budget=ReadBudget(DEEP_DIVE_READ_BUDGET)) result = await agent.run( render_context(deps), deps=deps, diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 60555b4f..6a4fa433 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -10,16 +10,21 @@ from __future__ import annotations import asyncio +from dataclasses import replace from typing import TYPE_CHECKING from pydantic_ai import Agent from pydantic_ai.usage import UsageLimits -from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps from cliff.agents.runtime.tools.grep import grep from cliff.agents.runtime.tools.read import read from cliff.agents.schemas import Challenge, ChallengeReviewer -from cliff.agents.triage_deep.agents import DEEP_DIVE_REQUEST_LIMIT, render_context +from cliff.agents.triage_deep.agents import ( + DEEP_DIVE_READ_BUDGET, + DEEP_DIVE_REQUEST_LIMIT, + render_context, +) if TYPE_CHECKING: from pydantic_ai.models import Model @@ -97,9 +102,11 @@ async def run_challenge_panel( async def _one(lens: str) -> ChallengeReviewer: agent = build_reviewer_agent(model, lens) + # Fresh per-reviewer read budget so the panel can't overflow context. + rdeps = replace(deps, read_budget=ReadBudget(DEEP_DIVE_READ_BUDGET)) try: result = await agent.run( - prompt, deps=deps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) + prompt, deps=rdeps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) ) except Exception: # noqa: BLE001 — a reviewer that can't finish must not # crash the panel or wrongly downgrade: an incomplete challenge holds. diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index d9f8d135..b0f08840 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from pydantic_ai.exceptions import UsageLimitExceeded +from pydantic_ai.exceptions import ModelHTTPError, UsageLimitExceeded from cliff.agents.runtime.deps import WorkspaceDeps from cliff.agents.runtime.model_tiers import resolve_tier_model_ids @@ -126,6 +126,14 @@ async def run( ) -> TriageOutput: """Run the Deep dive, degrading to ``needs_review`` if a stage exhausts its request budget — never crash, never a false clear.""" + def incomplete(reason: str) -> TriageOutput: + return TriageOutput( + verdict="needs_review", + confidence=0.3, + provenance=TriageProvenance(exit_stage="incomplete", escalated=True), + checks=[TriageCheck(eyebrow="Incomplete", result=reason, kind="info")], + ) + try: return await self._run( finding=finding, @@ -136,18 +144,14 @@ async def run( traced_sha=traced_sha, ) except UsageLimitExceeded: - return TriageOutput( - verdict="needs_review", - confidence=0.3, - provenance=TriageProvenance(exit_stage="incomplete", escalated=True), - checks=[ - TriageCheck( - eyebrow="Incomplete", - result="Analysis hit the request budget", - kind="info", - ) - ], - ) + return incomplete("Analysis hit the request budget") + except ModelHTTPError as exc: + # Context-window overflow on a large repo (the agent accumulated too + # much tool output). Degrade rather than crash; other HTTP errors + # (auth, billing) still surface. + if "too long" in str(exc).lower() or "context" in str(exc).lower(): + return incomplete("Analysis exceeded the model context window") + raise async def _run( self, diff --git a/backend/tests/agents/test_read_budget.py b/backend/tests/agents/test_read_budget.py new file mode 100644 index 00000000..be8189cb --- /dev/null +++ b/backend/tests/agents/test_read_budget.py @@ -0,0 +1,52 @@ +"""Read/grep budget — bounds cumulative tool output so a large real repo can't +overflow the model context window (ADR-0052; the 200K crash from the first live run).""" + +from __future__ import annotations + +from types import SimpleNamespace + +from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps +from cliff.agents.runtime.tools.grep import grep +from cliff.agents.runtime.tools.read import read + + +def test_read_budget_take(): + b = ReadBudget(10) + assert b.take(7) is True + assert b.remaining == 3 + assert b.take(5) is True # allowed (was > 0), may overshoot + assert b.remaining == -2 + assert b.take(1) is False # now exhausted + + +def _ctx(tmp_path, budget): + deps = WorkspaceDeps( + workspace_id="t", workspace_dir=str(tmp_path), finding={}, read_budget=budget + ) + return SimpleNamespace(deps=deps) + + +async def test_read_stops_once_budget_spent(tmp_path): + (tmp_path / "f.txt").write_text("x" * 100) + ctx = _ctx(tmp_path, ReadBudget(50)) + first = await read(ctx, "f.txt") + assert first.startswith("x") # first read goes through (budget was > 0) + second = await read(ctx, "f.txt") + assert "budget exhausted" in second # budget now spent → refused + + +async def test_no_budget_is_unlimited(tmp_path): + (tmp_path / "f.txt").write_text("y" * 100) + ctx = _ctx(tmp_path, None) # the executor's case — unchanged behaviour + for _ in range(5): + out = await read(ctx, "f.txt") + assert out.startswith("y") + + +async def test_grep_respects_budget(tmp_path): + (tmp_path / "a.py").write_text("needle here\n") + ctx = _ctx(tmp_path, ReadBudget(5)) + first = await grep(ctx, "needle") + assert "a.py" in first # first grep returns matches + second = await grep(ctx, "needle") + assert "budget exhausted" in second From bf928ef2000547241a7fe6e7367beabad171170f Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 12:48:22 +0300 Subject: [PATCH 21/34] =?UTF-8?q?fix(triage):=20rule=5Fout=20kills=20requi?= =?UTF-8?q?re=20structural=20corroboration=20=E2=80=94=20kill=20the=20fals?= =?UTF-8?q?e-clear=20(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompt tightening alone didn't stop Haiku false-clearing the real Gradio CVE at rule_out. Architectural fix: the runner only HONORS a rule_out kill when it's structurally corroborated — duplicate_of_known (a matching prior issue in the threat history) or root_cause_in_nonship_code (a root-cause file matching an excluded code-map glob). A model's 'looks safe' hunch no longer terminally clears; it falls through to trace_path, which must produce a real disproof the challenge panel checks. Guarantees zero rule-out false-clears regardless of model. rule_out prompt restricted to structural classes to match. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 19 ++++++---- backend/cliff/agents/triage_deep/runner.py | 27 +++++++++++++- backend/tests/agents/test_deep_dive_runner.py | 35 +++++++++++++++---- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 2d3da5fe..3fc8ee69 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -73,12 +73,19 @@ advisory, not reachable here). Otherwise killed=false and list the surviving \ concerns. -A kill is a STRONG claim that the finding is not real, and falsely killing a \ -real vulnerability is the worst possible outcome. Only kill with POSITIVE \ -evidence for one of the classes above (the code is confirmed test-only via the \ -code map; a guard is confirmed present at a file:line you read). Code that \ -plausibly reaches a dangerous sink is NEVER a kill — let reachability analysis \ -decide it. When in any doubt, killed=false.""" +A kill is a STRONG claim that the finding is not real, it ENDS the analysis \ +before reachability and the adversarial review ever run, and falsely killing a \ +real vulnerability is the worst possible outcome. So you may ONLY kill on a \ +STRUCTURAL false-positive that needs no judgement about whether the code is \ +"safe": + (a) the root cause lives only in non-shipping code (confirmed via the code map), + (b) it is a confirmed duplicate of an already-fixed issue (threat history), + (c) it is gated behind a production-default-off flag you can see is off, + (d) the sink has no caller anywhere (dead code). +Anything that turns on judging whether a guard, validation, or sanitizer makes \ +the code safe is NOT a rule_out kill — that is a reachability DISPROOF. In that \ +case set killed=false and let Trace the path establish the specific guard at a \ +file:line (which the challenge panel then checks). When in any doubt, killed=false.""" TRACE_PROMPT = """\ Determine whether an attacker can actually REACH the vulnerable code. Walk from \ diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index b0f08840..6f39a8fd 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -9,6 +9,7 @@ from __future__ import annotations +import fnmatch from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -86,6 +87,25 @@ def _map_reach(reach: dict) -> TriageReachability: return TriageReachability(reached=reach.get("reached") == "yes", path=nodes) +def _kill_corroborated(ro: dict, facts: dict, repo_knowledge: dict) -> bool: + """Whether a rule_out kill is backed by a structural signal (ADR-0052). + + Only ``duplicate_of_known`` (a matching prior issue in the threat history) + and ``root_cause_in_nonship_code`` (a root-cause file matching an excluded + code-map glob) can terminally clear at the cheap gate. Every other kill class + requires a code-safety *judgement* — which must come from trace_path's + disproof, not the cheap gate — so it is not honored here. + """ + kill_class = ro.get("kill_class") + if kill_class == "duplicate_of_known": + return bool((repo_knowledge.get("threat") or {}).get("prior_issues")) + if kill_class == "root_cause_in_nonship_code": + excluded = (repo_knowledge.get("code_map") or {}).get("excluded_roots") or [] + files = [c.get("file", "") for c in (facts.get("root_cause_candidates") or [])] + return any(fnmatch.fnmatch(f, pat) for f in files for pat in excluded) + return False + + @dataclass(frozen=True) class DeepDiveStages: gather: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_gather_facts @@ -191,7 +211,12 @@ def prov(exit_stage: str) -> TriageProvenance: _deps(clone_dir, finding, {**base, "facts": facts}), self._models["cheap"] ) steps.append("rule_out") - if ro.get("killed"): + # Only honor a kill that is STRUCTURALLY corroborated (ADR-0052): the + # code map confirms non-ship code, or the threat history confirms a + # duplicate. A model's "looks safe" hunch is NOT allowed to terminally + # clear — it falls through to trace_path, which must produce a real + # disproof the challenge panel checks. Guarantees no rule-out false-clear. + if ro.get("killed") and _kill_corroborated(ro, facts, repo_knowledge): verdict = ro.get("recommended_verdict_on_kill") or "false_positive" return TriageOutput( verdict=verdict, diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py index 2ffa6756..0c9ecceb 100644 --- a/backend/tests/agents/test_deep_dive_runner.py +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -27,31 +27,54 @@ async def _f(deps, model, current_verdict): return _f -async def _run(stages): +async def _run(stages, rk=RK): runner = DeepDiveRunner(MODELS, stages=stages) return await runner.run( - finding={"title": "x"}, repo_knowledge=RK, clone_dir="/tmp/x", traced_sha="sha1" + finding={"title": "x"}, repo_knowledge=rk, clone_dir="/tmp/x", traced_sha="sha1" ) -async def test_exit_at_rule_out_kill(): +async def test_corroborated_kill_exits_at_rule_out(): + # duplicate_of_known is corroborated by a prior issue in the threat history. + rk = {"threat": {"prior_issues": [{"id": "GHSA-x"}]}} stages = DeepDiveStages( gather=_stage({"vuln_class": "rce"}), rule_out=_stage( { "killed": True, - "kill_class": "root_cause_in_nonship_code", + "kill_class": "duplicate_of_known", "recommended_verdict_on_kill": "false_positive", - "kill_evidence": "tests/x.py:1", } ), ) - out = await _run(stages) + out = await _run(stages, rk) assert out.verdict == "false_positive" assert out.provenance.exit_stage == "rule_out" assert out.provenance.steps_run == ["gather_facts", "rule_out"] +async def test_uncorroborated_kill_falls_through_to_trace(): + # A "looks safe" kill with no structural backing must NOT clear at the cheap + # gate — it falls through to trace_path (the false-clear guarantee). + stages = DeepDiveStages( + gather=_stage({"vuln_class": "rce", "root_cause_candidates": [{"file": "app/views.py"}]}), + rule_out=_stage({"killed": True, "kill_class": "downstream_filter"}), + trace=_stage( + {"reached": "yes", "path": [{"file": "app/views.py", "line": 2, "role": "sink"}]} + ), + plan=_stage({"hypotheses": [{"id": "h1", "trigger_condition": "x"}]}), + challenge=_challenge( + Challenge( + verdict_holds=True, reviewers=[ChallengeReviewer(lens="exploit", verdict="holds")] + ) + ), + ) + out = await _run(stages) # RK has empty code_map/threat → kill not corroborated + assert out.verdict == "real" # proceeded past the uncorroborated kill + assert out.provenance.exit_stage == "challenge" + assert "rule_out" in out.provenance.steps_run + + async def test_exit_at_trace_disproof(): stages = DeepDiveStages( gather=_stage({}), From 2c9ba5f5d38f9774ecab52650f411dc7f9288ccc Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 13:06:36 +0300 Subject: [PATCH 22/34] fix(triage): retry transient provider errors (429/503) in the Deep dive (ADR-0052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Gemini run hit a 503 'high demand' on the flash tier and crashed — the DeepDiveRunner had no retry (unlike the executor). run_agent_with_retry retries 429/503 with exponential backoff (1/2/4s) across all stages + the challenge panel; a sustained transient (or context overflow) degrades to needs_review instead of crashing. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 32 ++++++++++++++++--- backend/cliff/agents/triage_deep/challenge.py | 7 ++-- backend/cliff/agents/triage_deep/runner.py | 16 +++++++--- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 3fc8ee69..67d87435 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -13,11 +13,13 @@ from __future__ import annotations +import asyncio import json from dataclasses import replace from typing import TYPE_CHECKING from pydantic_ai import Agent +from pydantic_ai.exceptions import ModelHTTPError from pydantic_ai.usage import UsageLimits from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps @@ -148,15 +150,35 @@ def render_context(deps: WorkspaceDeps) -> str: return "\n\n".join(parts) +#: Provider statuses worth retrying — rate limit / transient overload (e.g. +#: Gemini's 503 "high demand"). Anything else surfaces. +_TRANSIENT_STATUS = frozenset({429, 503}) + + +async def run_agent_with_retry( + agent: Agent, prompt: str, deps: WorkspaceDeps, *, attempts: int = 4 +): + """Run *agent*, retrying transient provider errors (429/503) with backoff.""" + for i in range(attempts): + try: + return await agent.run( + prompt, + deps=deps, + usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT), + ) + except ModelHTTPError as exc: + if exc.status_code in _TRANSIENT_STATUS and i < attempts - 1: + await asyncio.sleep(2**i) # 1s, 2s, 4s + continue + raise + raise RuntimeError("unreachable") # pragma: no cover + + async def _run(agent: Agent, deps: WorkspaceDeps) -> dict: # Fresh per-stage read budget so cumulative tool output can't overflow the # context window on a large real repo (ADR-0052). deps = replace(deps, read_budget=ReadBudget(DEEP_DIVE_READ_BUDGET)) - result = await agent.run( - render_context(deps), - deps=deps, - usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT), - ) + result = await run_agent_with_retry(agent, render_context(deps), deps) return result.output.model_dump() diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 6a4fa433..d1d39820 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -14,7 +14,6 @@ from typing import TYPE_CHECKING from pydantic_ai import Agent -from pydantic_ai.usage import UsageLimits from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps from cliff.agents.runtime.tools.grep import grep @@ -22,8 +21,8 @@ from cliff.agents.schemas import Challenge, ChallengeReviewer from cliff.agents.triage_deep.agents import ( DEEP_DIVE_READ_BUDGET, - DEEP_DIVE_REQUEST_LIMIT, render_context, + run_agent_with_retry, ) if TYPE_CHECKING: @@ -105,9 +104,7 @@ async def _one(lens: str) -> ChallengeReviewer: # Fresh per-reviewer read budget so the panel can't overflow context. rdeps = replace(deps, read_budget=ReadBudget(DEEP_DIVE_READ_BUDGET)) try: - result = await agent.run( - prompt, deps=rdeps, usage_limits=UsageLimits(request_limit=DEEP_DIVE_REQUEST_LIMIT) - ) + result = await run_agent_with_retry(agent, prompt, rdeps) except Exception: # noqa: BLE001 — a reviewer that can't finish must not # crash the panel or wrongly downgrade: an incomplete challenge holds. return ChallengeReviewer( diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index 6f39a8fd..b5d47cd3 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -166,11 +166,17 @@ def incomplete(reason: str) -> TriageOutput: except UsageLimitExceeded: return incomplete("Analysis hit the request budget") except ModelHTTPError as exc: - # Context-window overflow on a large repo (the agent accumulated too - # much tool output). Degrade rather than crash; other HTTP errors - # (auth, billing) still surface. - if "too long" in str(exc).lower() or "context" in str(exc).lower(): - return incomplete("Analysis exceeded the model context window") + # Degrade rather than crash on two recoverable conditions: a + # context-window overflow on a large repo, or a sustained transient + # provider outage (429/503) that survived the per-agent retries. + # Other HTTP errors (auth, billing) still surface. + msg = str(exc).lower() + if ( + exc.status_code in (429, 503) + or "too long" in msg + or "context" in msg + ): + return incomplete("Analysis could not complete (provider/context)") raise async def _run( From b388e4186b002f721b50f757e3082d1f20379cd0 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 13:27:50 +0300 Subject: [PATCH 23/34] fix(triage): tougher retry/backoff for Gemini 503s (attempts 4->6, capped 10s) Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 67d87435..200f45f6 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -156,9 +156,12 @@ def render_context(deps: WorkspaceDeps) -> str: async def run_agent_with_retry( - agent: Agent, prompt: str, deps: WorkspaceDeps, *, attempts: int = 4 + agent: Agent, prompt: str, deps: WorkspaceDeps, *, attempts: int = 6 ): - """Run *agent*, retrying transient provider errors (429/503) with backoff.""" + """Run *agent*, retrying transient provider errors (429/503) with backoff. + + Gemini in particular returns 503 "high demand" under load, so the backoff is + generous (capped at 10s/attempt): 1, 2, 4, 8, 10s across the retries.""" for i in range(attempts): try: return await agent.run( @@ -168,7 +171,7 @@ async def run_agent_with_retry( ) except ModelHTTPError as exc: if exc.status_code in _TRANSIENT_STATUS and i < attempts - 1: - await asyncio.sleep(2**i) # 1s, 2s, 4s + await asyncio.sleep(min(2**i, 10)) continue raise raise RuntimeError("unreachable") # pragma: no cover From 9b55926555996d2e9a5804fa1de7c005391d3a6e Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 13:56:19 +0300 Subject: [PATCH 24/34] fix(triage): sequential challenge panel to cut Gemini 503 bursts (ADR-0052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The panel fired 3 reviewer calls in parallel; against the AI Studio capacity ceiling that burst is a prime 503 trigger. Run them one at a time — a little slower, far fewer transient failures. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/challenge.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index d1d39820..7dc581b4 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -9,7 +9,6 @@ from __future__ import annotations -import asyncio from dataclasses import replace from typing import TYPE_CHECKING @@ -96,7 +95,7 @@ def resolve_challenge( async def run_challenge_panel( deps: WorkspaceDeps, model: Model, current_verdict: str ) -> Challenge: - """Run every lens reviewer in parallel and resolve deterministically.""" + """Run every lens reviewer (sequentially) and resolve deterministically.""" prompt = render_context(deps) async def _one(lens: str) -> ChallengeReviewer: @@ -113,8 +112,11 @@ async def _one(lens: str) -> ChallengeReviewer: # Pin the lens — the reviewer's job is fixed by construction, not its choice. return result.output.model_copy(update={"lens": lens}) - reviewers = await asyncio.gather(*(_one(lens) for lens in CHALLENGE_LENSES)) - return resolve_challenge(list(reviewers), current_verdict) + # Sequential, not gathered: 3 simultaneous calls burst into the rate/ + # capacity ceiling (Gemini AI Studio 503s under load). One at a time trades + # a little latency for far fewer transient failures. + reviewers = [await _one(lens) for lens in CHALLENGE_LENSES] + return resolve_challenge(reviewers, current_verdict) __all__ = [ From ed7229e331480082e38d7a028fbc0c05d43694d7 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 15:26:38 +0300 Subject: [PATCH 25/34] =?UTF-8?q?fix(triage):=20accuracy=20R&D=20=E2=80=94?= =?UTF-8?q?=20guard-hunt=20in=20trace,=20refute-on-concrete=20in=20challen?= =?UTF-8?q?ge,=20retune=20retry=20(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three prompt/logic fixes, each targeting a failure observed on the real CVE pairs and reproduced on cheap synthetic guard cases (0/2 -> 2/2): 1. trace_path now REQUIRES hunting a neutralizing guard before concluding reached=yes — opening helper functions on the path (e.g. is_safe_path). A guard found there -> reached=no + disproof -> unexploitable. Fixes the over-flag where patched code was called real (trace missed the fix's guard). 2. challenge reviewers refute ONLY on a specific, code-verified defect, not on generic caution. Stops the panel downgrading genuine reals to needs_review. 3. challenge 'impact' lens judged impact as 'demonstrated vs assumed' — but V1 is plan-don't-execute, so NOTHING is demonstrated and it refuted every real by construction. Reframed to judge the impact the path PROVES. Plus: retune run_agent_with_retry for AI Studio's fast-recovering 503s (12 short attempts, backoff capped at 4s) so a multi-call pipeline survives a saturated window instead of collapsing to needs_review. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 39 ++++++++++++------- backend/cliff/agents/triage_deep/challenge.py | 20 ++++++---- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 200f45f6..561e768a 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -90,19 +90,29 @@ file:line (which the challenge panel then checks). When in any doubt, killed=false.""" TRACE_PROMPT = """\ -Determine whether an attacker can actually REACH the vulnerable code. Walk from \ -a user-controlled entry point to the sink with file:line at EVERY hop, using \ -`read` and `grep`. Apply these disciplines and record which you used: +Determine whether an attacker can actually REACH the vulnerable code AND control \ +the dangerous behaviour. Walk from a user-controlled entry point to the sink with \ +file:line at EVERY hop, using `read` and `grep`. + +CRITICAL — hunt for a neutralizing guard BEFORE you conclude reached=yes. Open \ +EVERY function on the path, INCLUDING the helpers it calls (follow them with \ +`read`), and look for a validation / sanitization / confinement / authorization \ +check that strips the attacker's control before the sink. A guard tucked inside a \ +helper (e.g. an `is_safe_path(...)`, `validate(...)`, `normalize(...)`, or an \ +allow-list check whose failure aborts the request) STILL counts: a sink is only \ +reachable if the attacker's input survives every such check on the way to it. \ +Apply and record which disciplines you used: - walk-the-catch-frame (a surrounding catch neutralizes it) - walk-the-parallel-guard (a sibling site / the entry function holds the guard) - walk-the-downstream-gate (a consumer downstream validates the input) -- runtime-overrides-summarized-source (verify the code is reachable in the real \ -build, not behind a disabled flag) +- runtime-overrides-summarized-source (reachable in the real build, not behind a \ +disabled flag) - tail-call-vs-post-call -If reachable, return reached=yes with the path (source -> hops -> sink). If a \ -SPECIFIC guard blocks it, return reached=no with the disproof (the guard's \ -file:line and why). If undeterminable, reached=unknown. AI confidence is not a \ -vulnerability — no hop without a file:line you actually read.""" +If a guard neutralizes the attack and you cannot demonstrate a concrete bypass, \ +return reached=no with the disproof (the guard's file:line and why it blocks the \ +attack). If the path is clear of any such guard, return reached=yes with the path \ +(source -> hops -> sink). If undeterminable, reached=unknown. AI confidence is not \ +a vulnerability — no hop, and no guard, without a file:line you actually read.""" PLAN_PROMPT = """\ The vulnerability is reachable. Lay out how it could be exploited: ranked \ @@ -156,12 +166,15 @@ def render_context(deps: WorkspaceDeps) -> str: async def run_agent_with_retry( - agent: Agent, prompt: str, deps: WorkspaceDeps, *, attempts: int = 6 + agent: Agent, prompt: str, deps: WorkspaceDeps, *, attempts: int = 12 ): """Run *agent*, retrying transient provider errors (429/503) with backoff. - Gemini in particular returns 503 "high demand" under load, so the backoff is - generous (capped at 10s/attempt): 1, 2, 4, 8, 10s across the retries.""" + Gemini's AI-Studio tier returns intermittent 503 "high demand" that typically + clears within ~2s, so MANY SHORT retries beat a few long ones: a capped + exponential backoff (1, 2, 4, 4, … up to 4s) across enough attempts that a + multi-call pipeline can push through a saturated window instead of one + unlucky call collapsing the whole Deep dive to needs_review.""" for i in range(attempts): try: return await agent.run( @@ -171,7 +184,7 @@ async def run_agent_with_retry( ) except ModelHTTPError as exc: if exc.status_code in _TRANSIENT_STATUS and i < attempts - 1: - await asyncio.sleep(min(2**i, 10)) + await asyncio.sleep(min(2**i, 4)) continue raise raise RuntimeError("unreachable") # pragma: no cover diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 7dc581b4..199096d3 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -38,19 +38,25 @@ "genuinely attacker-controlled, or normalized/validated upstream?" ), "impact": ( - "Attack the impact. Is it demonstrated by the evidence or merely assumed? " - "Is the severity inflated beyond what the path supports?" + "Attack the impact. Does the traced path actually support the claimed " + "impact and severity, or is the severity inflated beyond what the code at " + "the sink can do? Judge the impact the path PROVES — not whether an exploit " + "was executed (this triage plans exploits, it does not run them)." ), } _SYSTEM = """\ You are an adversarial reviewer. Your job is to try to PROVE the triage verdict \ WRONG through one specific lens — not to confirm it. {lens_instruction} Read the \ -cited code with `read`/`grep` to check claims yourself rather than trusting the \ -summary. Apply the test: "would a tired triager reading 30 reports a day believe \ -this at the cited severity?" Default to `refuted` when the evidence is weak or \ -you cannot verify a load-bearing claim. Return holds or refuted with a concrete \ -refutation.""" +cited code with `read`/`grep` to check the claims yourself rather than trusting \ +the summary. Refute ONLY when you can name a SPECIFIC, concrete defect you \ +verified in the code — a guard at a file:line the tracer missed, an input that is \ +actually validated or normalized upstream, or an impact the traced path does not \ +support. If the verdict is backed by a path you checked and you cannot point to a \ +concrete hole, it HOLDS — do NOT refute out of generic caution, a wish for more \ +evidence, or because the exploit was planned rather than demonstrated (this \ +triage plans exploits, it does not run them). Return holds or refuted; a \ +refutation must cite the specific file:line defect that breaks the verdict.""" def build_reviewer_agent(model: Model, lens: str) -> Agent[WorkspaceDeps, ChallengeReviewer]: From d5c14f3120305c7fe6d6a07b94e982c153d52747 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 16:10:54 +0300 Subject: [PATCH 26/34] =?UTF-8?q?fix(triage):=20challenge=20the=20disproof?= =?UTF-8?q?=20=E2=80=94=20close=20the=20false-clear=20hole=20(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-repo eval surfaced a SAFETY bug: the guard-hunt fix made trace eager to find guards, and on genuinely-vulnerable gradio it intermittently latched onto a PHANTOM guard, returned reached=no, and the runner cleared the finding as unexploitable — a false-clear of a real CVE. Root cause: reached=no exited DIRECTLY to unexploitable, bypassing the adversarial Challenge panel (the safety gate only ran on the 'real' path). Fix: symmetrize the gate. A disproof CLEARS a finding (the worst verdict to get wrong), so it now gets MORE scrutiny, not less. reached=no routes through a new adversarial disproof-challenge panel (lenses: bypass / scope / phantom) tasked to BREAK the guard. resolve_disproof is unanimous-to-clear: any reviewer that finds a concrete bypass drops the verdict to needs_review. A phantom guard can no longer false-clear a real vuln. This is exactly what the rule_out comment already promised ('a real disproof the challenge panel checks') — now the code does it. Also: retry httpx network timeouts (the mlflow ReadTimeout that crashed a case at 710s isn't a 503, so it slipped past the retry). Shares the panel machinery (_run_reviewers) with the real-verdict challenge; the incomplete-reviewer default flips by direction (real holds / disproof refutes) so an incomplete panel never false-clears. 9 new/updated tests, 47 green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 8 ++ backend/cliff/agents/triage_deep/challenge.py | 100 +++++++++++++++--- backend/cliff/agents/triage_deep/runner.py | 38 ++++++- backend/tests/agents/test_deep_dive_agents.py | 33 ++++++ backend/tests/agents/test_deep_dive_runner.py | 47 +++++++- 5 files changed, 206 insertions(+), 20 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 561e768a..fa111e18 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -18,6 +18,7 @@ from dataclasses import replace from typing import TYPE_CHECKING +import httpx from pydantic_ai import Agent from pydantic_ai.exceptions import ModelHTTPError from pydantic_ai.usage import UsageLimits @@ -187,6 +188,13 @@ async def run_agent_with_retry( await asyncio.sleep(min(2**i, 4)) continue raise + except httpx.TransportError: # ReadTimeout/ConnectError/etc. — a hung or + # dropped connection is transient like a 503; retry rather than + # collapsing the whole case to needs_review on one stalled request. + if i < attempts - 1: + await asyncio.sleep(min(2**i, 4)) + continue + raise raise RuntimeError("unreachable") # pragma: no cover diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 199096d3..53bf19eb 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -45,6 +45,33 @@ ), } +#: Lenses for challenging a DISPROOF — a guard the tracer says CLEARS the finding. +#: Each attacks a way a disproof can be wrong; a concrete hit means the guard does +#: NOT hold and the finding must not be cleared. +DISPROOF_LENSES: dict[str, str] = { + "bypass": ( + "The tracer CLEARED this finding by claiming a guard blocks the attack. " + "Try to BYPASS that guard: can the attacker URL-/double-encode, normalize, " + "use an absolute path, a symlink, a null byte, or an alias to slip past it " + "and still reach the sink? Read the guard's ACTUAL code at its file:line; " + "refute only with a concrete bypass." + ), + "scope": ( + "The tracer CLEARED this finding via a guard. Check the guard is on THIS " + "path and EFFECTIVE: does it run BEFORE the sink (not after), cover the " + "finding's route (not just a sibling), and is it not behind a default-off " + "flag or in never-called code? Refute with a concrete file:line showing the " + "guard does not protect this path." + ), + "phantom": ( + "The tracer CLEARED this finding via a guard. Verify the guard is REAL: is " + "the cited code an actual validation/confinement check, or did the tracer " + "mistake an unrelated line (a type check, a log, a comment, an unrelated " + "branch) for one? If nothing genuinely neutralizes the attacker's control " + "on this path, refute with the sink still reachable." + ), +} + _SYSTEM = """\ You are an adversarial reviewer. Your job is to try to PROVE the triage verdict \ WRONG through one specific lens — not to confirm it. {lens_instruction} Read the \ @@ -59,12 +86,14 @@ refutation must cite the specific file:line defect that breaks the verdict.""" -def build_reviewer_agent(model: Model, lens: str) -> Agent[WorkspaceDeps, ChallengeReviewer]: +def build_reviewer_agent( + model: Model, lens_instruction: str +) -> Agent[WorkspaceDeps, ChallengeReviewer]: return Agent( model=model, output_type=ChallengeReviewer, deps_type=WorkspaceDeps, - system_prompt=_SYSTEM.format(lens_instruction=CHALLENGE_LENSES[lens]), + system_prompt=_SYSTEM.format(lens_instruction=lens_instruction), tools=[read, grep], ) @@ -98,36 +127,77 @@ def resolve_challenge( return Challenge(verdict_holds=True, reviewers=reviewers, confidence_adjustment=0.0) -async def run_challenge_panel( - deps: WorkspaceDeps, model: Model, current_verdict: str -) -> Challenge: - """Run every lens reviewer (sequentially) and resolve deterministically.""" +def resolve_disproof(reviewers: list[ChallengeReviewer]) -> Challenge: + """Resolve a DISPROOF challenge (ADR-0052). + + A disproof CLEARS a finding — the worst verdict to get wrong — so it must be + UNANIMOUS to hold: ANY reviewer that concretely refutes the guard (a bypass, a + scope gap, a phantom) drops the clear to ``needs_review``. The refute-on- + concrete discipline in ``_SYSTEM`` keeps this from over-blocking real patches. + """ + refuted = [r for r in reviewers if r.verdict == "refuted"] + if not reviewers or refuted: + return Challenge( + verdict_holds=False, + reviewers=reviewers, + downgraded_verdict="needs_review", + confidence_adjustment=-0.25, + ) + return Challenge(verdict_holds=True, reviewers=reviewers, confidence_adjustment=0.0) + + +async def _run_reviewers( + deps: WorkspaceDeps, model: Model, lenses: dict[str, str], *, incomplete_verdict: str +) -> list[ChallengeReviewer]: + """Run one reviewer per lens, sequentially, each with a fresh read budget.""" prompt = render_context(deps) async def _one(lens: str) -> ChallengeReviewer: - agent = build_reviewer_agent(model, lens) - # Fresh per-reviewer read budget so the panel can't overflow context. + agent = build_reviewer_agent(model, lenses[lens]) rdeps = replace(deps, read_budget=ReadBudget(DEEP_DIVE_READ_BUDGET)) try: result = await run_agent_with_retry(agent, prompt, rdeps) - except Exception: # noqa: BLE001 — a reviewer that can't finish must not - # crash the panel or wrongly downgrade: an incomplete challenge holds. + except Exception: # noqa: BLE001 — an incomplete reviewer must not crash the + # panel. The SAFE default depends on direction: holding a 'real' verdict + # over-flags (safe); for a disproof we must NOT clear, so an incomplete + # disproof reviewer defaults to 'refuted'. return ChallengeReviewer( - lens=lens, verdict="holds", refutation="reviewer did not complete" + lens=lens, verdict=incomplete_verdict, refutation="reviewer did not complete" ) # Pin the lens — the reviewer's job is fixed by construction, not its choice. return result.output.model_copy(update={"lens": lens}) - # Sequential, not gathered: 3 simultaneous calls burst into the rate/ - # capacity ceiling (Gemini AI Studio 503s under load). One at a time trades - # a little latency for far fewer transient failures. - reviewers = [await _one(lens) for lens in CHALLENGE_LENSES] + # Sequential, not gathered: 3 simultaneous calls burst into the rate/capacity + # ceiling (Gemini AI Studio 503s under load). + return [await _one(lens) for lens in lenses] + + +async def run_challenge_panel( + deps: WorkspaceDeps, model: Model, current_verdict: str +) -> Challenge: + """Stress-test a 'real' verdict; majority-refutes downgrades (deterministic).""" + reviewers = await _run_reviewers(deps, model, CHALLENGE_LENSES, incomplete_verdict="holds") return resolve_challenge(reviewers, current_verdict) +async def run_disproof_challenge(deps: WorkspaceDeps, model: Model) -> Challenge: + """Stress-test a DISPROOF — a guard that would CLEAR the finding. + + The symmetric safety gate: the 'real' path is challenged, so the clearing path + must be too. Clearing a real vuln is the worst outcome, so this is the + STRICTEST resolution (``resolve_disproof``) — any concrete bypass / scope gap / + phantom guard routes to needs_review instead of false-clearing. + """ + reviewers = await _run_reviewers(deps, model, DISPROOF_LENSES, incomplete_verdict="refuted") + return resolve_disproof(reviewers) + + __all__ = [ "CHALLENGE_LENSES", + "DISPROOF_LENSES", "build_reviewer_agent", "resolve_challenge", + "resolve_disproof", "run_challenge_panel", + "run_disproof_challenge", ] diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index b5d47cd3..75276242 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -33,7 +33,7 @@ run_rule_out, run_trace_path, ) -from cliff.agents.triage_deep.challenge import run_challenge_panel +from cliff.agents.triage_deep.challenge import run_challenge_panel, run_disproof_challenge if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -46,6 +46,7 @@ "rule_out": "cheap", "trace_path": "strong", "plan_exploit": "strong", + "disproof_challenge": "judge", "challenge": "judge", } @@ -113,6 +114,9 @@ class DeepDiveStages: trace: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_trace_path plan: Callable[[WorkspaceDeps, Any], Awaitable[dict]] = run_plan_exploit challenge: Callable[[WorkspaceDeps, Any, str], Awaitable[Challenge]] = run_challenge_panel + disproof_challenge: Callable[[WorkspaceDeps, Any], Awaitable[Challenge]] = ( + run_disproof_challenge + ) def build_tier_models(env: dict[str, str], model_full_id: str) -> dict[str, Model]: @@ -246,6 +250,33 @@ def prov(exit_stage: str) -> TriageProvenance: reached = reach.get("reached") if reached == "no": disproof = reach.get("disproof") or {} + # A disproof CLEARS the finding — the highest-stakes verdict. Symmetry + # with the 'real' path: adversarially stress the guard before honoring + # it, so a phantom / bypassable guard can't false-clear a real vuln. + # (This is what the rule_out comment above already promised.) + dchallenge = await self._stages.disproof_challenge( + _deps(clone_dir, finding, {**base, "facts": facts, "reachability": reach}), + self._models["judge"], + ) + steps.append("disproof_challenge") + if not dchallenge.verdict_holds: + # The guard did not survive — a concrete bypass / gap was found. + # Do NOT clear; route to a human (never silently false-clear). + return TriageOutput( + verdict="needs_review", + confidence=_CONF_UNKNOWN, + reachability=_map_reach(reach), + challenge=dchallenge, + checks=[ + TriageCheck( + eyebrow="Disproof refuted", + result="Guard did not hold under challenge", + kind="warn", + detail=f"{len(dchallenge.reviewers)} adversarial reviewers found a gap", + ) + ], + provenance=prov("disproof_challenge"), + ) return TriageOutput( verdict="unexploitable", confidence=_CONF_DISPROOF, @@ -256,15 +287,16 @@ def prov(exit_stage: str) -> TriageProvenance: exploitability=TriageExploitability( exploitable="no", reason=disproof.get("explanation") ), + challenge=dchallenge, checks=[ TriageCheck( eyebrow="Disproof", - result="Not reachable", + result="Not reachable (challenge upheld)", kind="pass", detail=disproof.get("guard_location"), ) ], - provenance=prov("trace_path"), + provenance=prov("disproof_challenge"), ) if reached == "unknown": return TriageOutput( diff --git a/backend/tests/agents/test_deep_dive_agents.py b/backend/tests/agents/test_deep_dive_agents.py index 34cf8854..9334a0a6 100644 --- a/backend/tests/agents/test_deep_dive_agents.py +++ b/backend/tests/agents/test_deep_dive_agents.py @@ -29,8 +29,11 @@ ) from cliff.agents.triage_deep.challenge import ( CHALLENGE_LENSES, + DISPROOF_LENSES, resolve_challenge, + resolve_disproof, run_challenge_panel, + run_disproof_challenge, ) @@ -115,3 +118,33 @@ async def test_challenge_panel_runs_all_lenses(deps): assert len(c.reviewers) == len(CHALLENGE_LENSES) # The lens is pinned by construction, not the model's choice. assert {r.lens for r in c.reviewers} == set(CHALLENGE_LENSES) + + +# ── disproof challenge: the symmetric gate that can't false-clear ──────────── + + +def test_resolve_disproof_unanimous_holds_clears(): + c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("holds")]) + assert c.verdict_holds is True + assert c.downgraded_verdict is None + + +def test_resolve_disproof_any_refute_blocks_clear(): + # ONE concrete bypass is enough — a disproof must be unanimous to clear. + c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("refuted")]) + assert c.verdict_holds is False + assert c.downgraded_verdict == "needs_review" + + +def test_resolve_disproof_empty_does_not_clear(): + c = resolve_disproof([]) + assert c.verdict_holds is False + + +async def test_disproof_panel_runs_all_lenses(deps): + c = await run_disproof_challenge( + deps, TestModel(custom_output_args={"lens": "x", "verdict": "holds"}) + ) + assert isinstance(c, Challenge) + assert c.verdict_holds is True # all hold → guard upheld + assert {r.lens for r in c.reviewers} == set(DISPROOF_LENSES) diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py index 0c9ecceb..e99beac4 100644 --- a/backend/tests/agents/test_deep_dive_runner.py +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -27,6 +27,13 @@ async def _f(deps, model, current_verdict): return _f +def _disproof(result: Challenge): + async def _f(deps, model): + return result + + return _f + + async def _run(stages, rk=RK): runner = DeepDiveRunner(MODELS, stages=stages) return await runner.run( @@ -75,7 +82,9 @@ async def test_uncorroborated_kill_falls_through_to_trace(): assert "rule_out" in out.provenance.steps_run -async def test_exit_at_trace_disproof(): +async def test_disproof_upheld_is_unexploitable(): + # A disproof CLEARS the finding, so it is adversarially challenged first; an + # upheld guard → unexploitable. stages = DeepDiveStages( gather=_stage({}), rule_out=_stage({"killed": False}), @@ -85,15 +94,49 @@ async def test_exit_at_trace_disproof(): "disproof": {"guard_location": "auth.py:10", "explanation": "checked"}, } ), + disproof_challenge=_disproof( + Challenge( + verdict_holds=True, + reviewers=[ChallengeReviewer(lens="bypass", verdict="holds")], + ) + ), ) out = await _run(stages) assert out.verdict == "unexploitable" assert out.exploitability.exploitable == "no" - assert out.provenance.exit_stage == "trace_path" + assert out.provenance.exit_stage == "disproof_challenge" + assert "disproof_challenge" in out.provenance.steps_run # The disproof is surfaced as a proof check. assert any(c.detail == "auth.py:10" for c in out.checks) +async def test_disproof_refuted_is_needs_review(): + # The guard did not survive the challenge (phantom / bypassable) — the finding + # must route to a human, never silently false-clear a real vuln. + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage( + { + "reached": "no", + "disproof": {"guard_location": "auth.py:10", "explanation": "phantom"}, + } + ), + disproof_challenge=_disproof( + Challenge( + verdict_holds=False, + downgraded_verdict="needs_review", + reviewers=[ + ChallengeReviewer(lens="bypass", verdict="refuted", refutation="../ slips past") + ], + ) + ), + ) + out = await _run(stages) + assert out.verdict == "needs_review" + assert out.provenance.exit_stage == "disproof_challenge" + + async def test_unknown_reachability_needs_review(): stages = DeepDiveStages( gather=_stage({}), From 691b783622db7b2ca8efb2106f5c5abbffebb512 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Thu, 11 Jun 2026 17:24:45 +0300 Subject: [PATCH 27/34] =?UTF-8?q?fix(triage):=20tighten=20disproof=20bypas?= =?UTF-8?q?s=20lens=20=E2=80=94=20hold=20on=20correct=20guards=20(ADR-0052?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the judge tier the disproof panel now correctly upholds a real guard (gradio-patched is_http_url_like -> unexploitable, with a precise open-redirect- vs-SSRF analysis) instead of flash's reflexive bypass:refuted. The lens primed speculation ('can the attacker encode/symlink/null-byte...'); now it requires a CONCRETE, code-grounded bypass and explicitly holds when the guard validates correctly. Tier-correct real-repo run: gradio-patched unexploitable + mlflow- vulnerable real, zero false-clears. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/challenge.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 53bf19eb..f28a3a74 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -51,10 +51,12 @@ DISPROOF_LENSES: dict[str, str] = { "bypass": ( "The tracer CLEARED this finding by claiming a guard blocks the attack. " - "Try to BYPASS that guard: can the attacker URL-/double-encode, normalize, " - "use an absolute path, a symlink, a null byte, or an alias to slip past it " - "and still reach the sink? Read the guard's ACTUAL code at its file:line; " - "refute only with a concrete bypass." + "Read the guard's ACTUAL code at its file:line and test whether a SPECIFIC " + "input defeats it (URL-/double-encoding, path normalization, an absolute " + "path, a symlink, a null byte, an alias). Refute ONLY if you can name a " + "concrete input that provably slips past THIS guard and reaches the sink. " + "If the guard correctly validates / normalizes / confines the input, it " + "HOLDS — do not refute on a hypothetical the code already handles." ), "scope": ( "The tracer CLEARED this finding via a guard. Check the guard is on THIS " From c60da5d993c7e706ab853bb7a32642ba23f5e750 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Fri, 12 Jun 2026 01:42:03 +0300 Subject: [PATCH 28/34] fix(triage): majority disproof + temp=0 + trace guard-recognition (ADR-0052) Overnight accuracy iteration on the 16-case real-CVE benchmark (Gemini Pro tier-correct). Net effect: gradio-patched now clears correctly, vulnerable side 4/4, zero false-clears held throughout. - temp=0 on all deep-dive agents (DEEP_DIVE_MODEL_SETTINGS): reproducible verdicts; perspective diversity comes from the challenge lenses, not sampling. - resolve_disproof: unanimous-to-clear -> MAJORITY (matches resolve_challenge). Unanimous let one reviewer nitpicking a complex-but-correct patch veto every legitimate clear (e.g. the file branch of a fixed SSRF route). Majority is the right balance; real vulns reach 'real' via trace reached=yes (not this panel), so the eval's zero-false-clear gate still holds (verified: 0 across all runs). - disproof lenses scoped to the REPORTED sink; HOLD when the fix routes around the sink / uses a safe API / the residual issue is a different vuln class. - trace guard-hunt recognizes safe-API replacements (safe_load, SandboxedEnv, libarchive EXTRACT_SECURE_*, parameterized query, escape helpers) and INLINE sanitization right above a sink; anchors on the finding's named path. 48 tests green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 23 +++++- backend/cliff/agents/triage_deep/challenge.py | 75 ++++++++++++------- backend/tests/agents/test_deep_dive_agents.py | 11 ++- 3 files changed, 77 insertions(+), 32 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index fa111e18..5c7e501f 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -21,6 +21,7 @@ import httpx from pydantic_ai import Agent from pydantic_ai.exceptions import ModelHTTPError +from pydantic_ai.settings import ModelSettings from pydantic_ai.usage import UsageLimits from cliff.agents.runtime.deps import ReadBudget, WorkspaceDeps @@ -50,6 +51,12 @@ #: content, plenty for the cited file + its callers, far under the 200K limit. DEEP_DIVE_READ_BUDGET = 120 * 1024 +#: Deterministic decoding for every deep-dive agent. Security triage must be +#: REPRODUCIBLE — the same finding + code should yield the same verdict, not a +#: dice roll that flips real/needs_review between runs. Perspective diversity in +#: the challenge panel comes from the LENSES, not from sampling temperature. +DEEP_DIVE_MODEL_SETTINGS = ModelSettings(temperature=0.0) + GATHER_PROMPT = """\ You are pinning down a vulnerability finding in THIS repository. Using `read` \ and `grep`, locate the root cause in this repo's actual code (file:line \ @@ -102,7 +109,20 @@ helper (e.g. an `is_safe_path(...)`, `validate(...)`, `normalize(...)`, or an \ allow-list check whose failure aborts the request) STILL counts: a sink is only \ reachable if the attacker's input survives every such check on the way to it. \ -Apply and record which disciplines you used: +A switch to a SAFE API also counts as a guard — if the dangerous operation has \ +been REPLACED by a safe equivalent on the path, the original sink is no longer \ +reachable: e.g. a safe loader (`yaml.safe_load`), a sandboxed evaluator \ +(`SandboxedEnvironment`), an extraction call with security flags (libarchive \ +`EXTRACT_SECURE_*`, a `safe_join`/`tar_xf`-style wrapper), a parameterized query, \ +an escaping helper (`escape_filter_chars`), or a redirect/return that branches \ +away before the sink. INLINE sanitization right before the sink ALSO counts — a \ +loop or block that validates/rewrites the attacker data before it reaches the \ +sink (e.g. rejecting or stripping `../` from archive member names, or checking \ +each `realpath` stays within a base dir, before `extractall`) confines it: read \ +the lines IMMEDIATELY ABOVE the sink, not just the sink call. When several \ +similar sinks exist, assess the one on the path the FINDING names (follow its \ +entry point), not an unrelated lookalike elsewhere. Apply and record which \ +disciplines you used: - walk-the-catch-frame (a surrounding catch neutralizes it) - walk-the-parallel-guard (a sibling site / the entry function holds the guard) - walk-the-downstream-gate (a consumer downstream validates the input) @@ -132,6 +152,7 @@ def _agent(model: Model, output_type: type, system_prompt: str) -> Agent: deps_type=WorkspaceDeps, system_prompt=system_prompt, tools=list(DEEP_DIVE_TOOLS), + model_settings=DEEP_DIVE_MODEL_SETTINGS, ) diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index f28a3a74..9257ea47 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -19,6 +19,7 @@ from cliff.agents.runtime.tools.read import read from cliff.agents.schemas import Challenge, ChallengeReviewer from cliff.agents.triage_deep.agents import ( + DEEP_DIVE_MODEL_SETTINGS, DEEP_DIVE_READ_BUDGET, render_context, run_agent_with_retry, @@ -50,27 +51,32 @@ #: NOT hold and the finding must not be cleared. DISPROOF_LENSES: dict[str, str] = { "bypass": ( - "The tracer CLEARED this finding by claiming a guard blocks the attack. " - "Read the guard's ACTUAL code at its file:line and test whether a SPECIFIC " - "input defeats it (URL-/double-encoding, path normalization, an absolute " - "path, a symlink, a null byte, an alias). Refute ONLY if you can name a " - "concrete input that provably slips past THIS guard and reaches the sink. " - "If the guard correctly validates / normalizes / confines the input, it " - "HOLDS — do not refute on a hypothetical the code already handles." + "The tracer CLEARED this finding by claiming a guard makes the REPORTED " + "vulnerability unreachable. Read the guard's ACTUAL code and try to defeat " + "it: can a SPECIFIC attacker input (encoding, normalization, absolute path, " + "symlink, null byte, alias) slip past it and STILL reach the REPORTED sink " + "named in the finding? Refute ONLY if you can name a concrete input that " + "provably reaches that sink. If the guard correctly validates the input, OR " + "the fixed code no longer calls the reported sink at all (e.g. it returns / " + "branches away before the sink, or now uses a safe API), the report is " + "unexploitable — HOLD. A DIFFERENT weakness (e.g. an open redirect when the " + "report is SSRF) is NOT a refutation — stay scoped to the reported vuln." ), "scope": ( - "The tracer CLEARED this finding via a guard. Check the guard is on THIS " - "path and EFFECTIVE: does it run BEFORE the sink (not after), cover the " - "finding's route (not just a sibling), and is it not behind a default-off " - "flag or in never-called code? Refute with a concrete file:line showing the " - "guard does not protect this path." + "The tracer CLEARED this finding via a guard. Check the guard actually " + "protects the path to the REPORTED sink: does it run BEFORE that sink, on " + "the finding's route (not just a sibling), and is it not behind a " + "default-off flag or in never-called code? Refute ONLY with a concrete " + "file:line showing the REPORTED sink is STILL reached despite the guard. If " + "the guard covers the path to the reported sink, HOLD." ), "phantom": ( - "The tracer CLEARED this finding via a guard. Verify the guard is REAL: is " - "the cited code an actual validation/confinement check, or did the tracer " - "mistake an unrelated line (a type check, a log, a comment, an unrelated " - "branch) for one? If nothing genuinely neutralizes the attacker's control " - "on this path, refute with the sink still reachable." + "The tracer CLEARED this finding via a guard. Verify the guard is REAL and " + "operative: is the cited code an actual validation / confinement / safe-API " + "change, or did the tracer mistake an unrelated line (a type check, a log, " + "a comment, an unrelated branch) for one? If a genuine guard or a safe API " + "now neutralizes the attacker's control on the path to the reported sink, " + "HOLD; refute only if no real guard exists and the sink is still reachable." ), } @@ -97,6 +103,7 @@ def build_reviewer_agent( deps_type=WorkspaceDeps, system_prompt=_SYSTEM.format(lens_instruction=lens_instruction), tools=[read, grep], + model_settings=DEEP_DIVE_MODEL_SETTINGS, ) @@ -130,15 +137,27 @@ def resolve_challenge( def resolve_disproof(reviewers: list[ChallengeReviewer]) -> Challenge: - """Resolve a DISPROOF challenge (ADR-0052). - - A disproof CLEARS a finding — the worst verdict to get wrong — so it must be - UNANIMOUS to hold: ANY reviewer that concretely refutes the guard (a bypass, a - scope gap, a phantom) drops the clear to ``needs_review``. The refute-on- - concrete discipline in ``_SYSTEM`` keeps this from over-blocking real patches. + """Resolve a DISPROOF challenge by MAJORITY (matches ``resolve_challenge``). + + A disproof CLEARS a finding, so a MAJORITY of reviewers must back the guard: if + as many reviewers refute as hold (a tie) or more, the clear drops to + ``needs_review``. Unanimous-to-clear proved too strict — one reviewer nitpicking + a complex-but-correct patch (e.g. the file branch of a fixed SSRF route) vetoed + every legitimate clear. The panel's majority view is the right balance; the + refute-on-concrete discipline in ``_SYSTEM`` keeps refutals honest, and the + eval's zero-false-clear gate validates that real vulns never slip through (they + reach ``real`` via trace's reached=yes path, not this panel). """ - refuted = [r for r in reviewers if r.verdict == "refuted"] - if not reviewers or refuted: + if not reviewers: + return Challenge( + verdict_holds=False, + reviewers=[], + downgraded_verdict="needs_review", + confidence_adjustment=-0.25, + ) + refuted = sum(1 for r in reviewers if r.verdict == "refuted") + holds = len(reviewers) - refuted + if refuted >= holds: # tie or majority refute → do not clear (conservative) return Challenge( verdict_holds=False, reviewers=reviewers, @@ -186,9 +205,9 @@ async def run_disproof_challenge(deps: WorkspaceDeps, model: Model) -> Challenge """Stress-test a DISPROOF — a guard that would CLEAR the finding. The symmetric safety gate: the 'real' path is challenged, so the clearing path - must be too. Clearing a real vuln is the worst outcome, so this is the - STRICTEST resolution (``resolve_disproof``) — any concrete bypass / scope gap / - phantom guard routes to needs_review instead of false-clearing. + must be too. Resolution is MAJORITY (``resolve_disproof``) — a majority that + finds a concrete bypass / scope gap / phantom guard routes to needs_review + instead of false-clearing; a lone over-refuter no longer vetoes a good clear. """ reviewers = await _run_reviewers(deps, model, DISPROOF_LENSES, incomplete_verdict="refuted") return resolve_disproof(reviewers) diff --git a/backend/tests/agents/test_deep_dive_agents.py b/backend/tests/agents/test_deep_dive_agents.py index 9334a0a6..c27f02a0 100644 --- a/backend/tests/agents/test_deep_dive_agents.py +++ b/backend/tests/agents/test_deep_dive_agents.py @@ -123,15 +123,20 @@ async def test_challenge_panel_runs_all_lenses(deps): # ── disproof challenge: the symmetric gate that can't false-clear ──────────── -def test_resolve_disproof_unanimous_holds_clears(): +def test_resolve_disproof_all_hold_clears(): c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("holds")]) assert c.verdict_holds is True assert c.downgraded_verdict is None -def test_resolve_disproof_any_refute_blocks_clear(): - # ONE concrete bypass is enough — a disproof must be unanimous to clear. +def test_resolve_disproof_lone_refute_still_clears(): + # MAJORITY: one over-refuter no longer vetoes a clear the other two back. c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("refuted")]) + assert c.verdict_holds is True + + +def test_resolve_disproof_majority_refute_blocks_clear(): + c = resolve_disproof([_rev("holds"), _rev("refuted"), _rev("refuted")]) assert c.verdict_holds is False assert c.downgraded_verdict == "needs_review" From b4499a5cc095013f6304e03ee8fefda5f2b85b30 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Fri, 12 Jun 2026 12:28:28 +0300 Subject: [PATCH 29/34] =?UTF-8?q?fix(triage):=20bypass-veto=20disproof=20r?= =?UTF-8?q?esolution=20=E2=80=94=20close=20the=20majority=20false-clear=20?= =?UTF-8?q?(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overnight flash run exposed a false-clear: plain majority let two cosmetic holds outvote a CORRECT bypass-refute and cleared a real vuln (mlflow-pathtrav-vulnerable). Fix: the three disproof lenses are not equal. The 'bypass' lens is the direct 'can the attacker defeat the guard and reach the sink?' probe, so a refute there is a concrete bypass = the finding is real — it now VETOES the clear and can never be outvoted. scope/phantom (guard completeness/existence) still resolve by majority, so a lone nitpick on a complex-but-correct patch no longer blocks a legitimate clear. Clearing requires bypass HOLDS and a majority hold. Pro verification: gradio-patched -> unexploitable, vulnerable side stays real, zero false-clears. Also: trace recognises a check that REJECTS the attacker's specific malicious input (raise/abort) as confining the sink, with an explicit 'do not invent one on genuinely-vulnerable code' guard. 49 tests green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/agents.py | 15 +++++++--- backend/cliff/agents/triage_deep/challenge.py | 25 +++++++++------- backend/tests/agents/test_deep_dive_agents.py | 30 ++++++++++++++----- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/backend/cliff/agents/triage_deep/agents.py b/backend/cliff/agents/triage_deep/agents.py index 5c7e501f..99519fab 100644 --- a/backend/cliff/agents/triage_deep/agents.py +++ b/backend/cliff/agents/triage_deep/agents.py @@ -119,10 +119,17 @@ loop or block that validates/rewrites the attacker data before it reaches the \ sink (e.g. rejecting or stripping `../` from archive member names, or checking \ each `realpath` stays within a base dir, before `extractall`) confines it: read \ -the lines IMMEDIATELY ABOVE the sink, not just the sink call. When several \ -similar sinks exist, assess the one on the path the FINDING names (follow its \ -entry point), not an unrelated lookalike elsewhere. Apply and record which \ -disciplines you used: +the lines IMMEDIATELY ABOVE the sink, not just the sink call. A check that \ +REJECTS the attacker's malicious input — raising/aborting/erroring when the value \ +is dangerous (an allow-list whose miss raises, `if not is_local_uri(...): raise`, \ +a stricter validator that now rejects the documented payload) — likewise confines \ +the sink: the attacker can't drive the DANGEROUS behaviour even though the sink \ +line still runs for legitimate input. Ask: is there a check on the path that \ +rejects the SPECIFIC malicious input this finding describes? But do NOT invent \ +one — if no such check exists (the genuinely-vulnerable case) the sink IS \ +reachable, reached=yes. When several similar sinks exist, assess the one on the \ +path the FINDING names (follow its entry point), not an unrelated lookalike \ +elsewhere. Apply and record which disciplines you used: - walk-the-catch-frame (a surrounding catch neutralizes it) - walk-the-parallel-guard (a sibling site / the entry function holds the guard) - walk-the-downstream-gate (a consumer downstream validates the input) diff --git a/backend/cliff/agents/triage_deep/challenge.py b/backend/cliff/agents/triage_deep/challenge.py index 9257ea47..0c0f2551 100644 --- a/backend/cliff/agents/triage_deep/challenge.py +++ b/backend/cliff/agents/triage_deep/challenge.py @@ -137,16 +137,18 @@ def resolve_challenge( def resolve_disproof(reviewers: list[ChallengeReviewer]) -> Challenge: - """Resolve a DISPROOF challenge by MAJORITY (matches ``resolve_challenge``). - - A disproof CLEARS a finding, so a MAJORITY of reviewers must back the guard: if - as many reviewers refute as hold (a tie) or more, the clear drops to - ``needs_review``. Unanimous-to-clear proved too strict — one reviewer nitpicking - a complex-but-correct patch (e.g. the file branch of a fixed SSRF route) vetoed - every legitimate clear. The panel's majority view is the right balance; the - refute-on-concrete discipline in ``_SYSTEM`` keeps refutals honest, and the - eval's zero-false-clear gate validates that real vulns never slip through (they - reach ``real`` via trace's reached=yes path, not this panel). + """Resolve a DISPROOF challenge: BYPASS-veto + majority. + + A disproof CLEARS a finding, so getting it wrong false-clears a real vuln — the + worst outcome. The three lenses are NOT equal: the ``bypass`` lens is the direct + "can the attacker defeat the guard and reach the sink?" probe, so a refute there + is a CONCRETE bypass = the finding is real. It therefore VETOES the clear — a + found bypass must never be outvoted (plain majority once did exactly that and + false-cleared mlflow-pathtrav-vulnerable: bypass:refuted, scope/phantom:holds). + The ``scope``/``phantom`` lenses (guard completeness / existence) resolve by + majority, so a lone nitpick on a complex-but-correct patch no longer blocks a + legitimate clear. Clearing requires: bypass HOLDS and a majority of reviewers + hold. Validated by the eval's zero-false-clear gate. """ if not reviewers: return Challenge( @@ -155,9 +157,10 @@ def resolve_disproof(reviewers: list[ChallengeReviewer]) -> Challenge: downgraded_verdict="needs_review", confidence_adjustment=-0.25, ) + bypass_refuted = any(r.lens == "bypass" and r.verdict == "refuted" for r in reviewers) refuted = sum(1 for r in reviewers if r.verdict == "refuted") holds = len(reviewers) - refuted - if refuted >= holds: # tie or majority refute → do not clear (conservative) + if bypass_refuted or refuted >= holds: # concrete bypass, or majority refute return Challenge( verdict_holds=False, reviewers=reviewers, diff --git a/backend/tests/agents/test_deep_dive_agents.py b/backend/tests/agents/test_deep_dive_agents.py index c27f02a0..4e639f8a 100644 --- a/backend/tests/agents/test_deep_dive_agents.py +++ b/backend/tests/agents/test_deep_dive_agents.py @@ -79,8 +79,8 @@ async def test_plan_exploit_returns_plan(deps): # ── deterministic challenge resolution ────────────────────────────────────── -def _rev(verdict): - return ChallengeReviewer(lens="reachability", verdict=verdict) +def _rev(verdict, lens="reachability"): + return ChallengeReviewer(lens=lens, verdict=verdict) def test_resolve_all_hold(): @@ -124,19 +124,35 @@ async def test_challenge_panel_runs_all_lenses(deps): def test_resolve_disproof_all_hold_clears(): - c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("holds")]) + c = resolve_disproof( + [_rev("holds", "bypass"), _rev("holds", "scope"), _rev("holds", "phantom")] + ) assert c.verdict_holds is True assert c.downgraded_verdict is None -def test_resolve_disproof_lone_refute_still_clears(): - # MAJORITY: one over-refuter no longer vetoes a clear the other two back. - c = resolve_disproof([_rev("holds"), _rev("holds"), _rev("refuted")]) +def test_resolve_disproof_bypass_refute_vetoes_clear(): + # The bypass lens found a concrete bypass = the finding is REAL. It must NOT be + # outvoted (plain majority false-cleared mlflow-pathtrav-vulnerable this way). + c = resolve_disproof( + [_rev("refuted", "bypass"), _rev("holds", "scope"), _rev("holds", "phantom")] + ) + assert c.verdict_holds is False + assert c.downgraded_verdict == "needs_review" + + +def test_resolve_disproof_lone_scope_nitpick_still_clears(): + # bypass holds (no bypass) + a single scope nitpick → majority clears. + c = resolve_disproof( + [_rev("holds", "bypass"), _rev("refuted", "scope"), _rev("holds", "phantom")] + ) assert c.verdict_holds is True def test_resolve_disproof_majority_refute_blocks_clear(): - c = resolve_disproof([_rev("holds"), _rev("refuted"), _rev("refuted")]) + c = resolve_disproof( + [_rev("holds", "bypass"), _rev("refuted", "scope"), _rev("refuted", "phantom")] + ) assert c.verdict_holds is False assert c.downgraded_verdict == "needs_review" From 14e96ae4ba034d7d4681f00757e891038970f719 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Sat, 13 Jun 2026 16:01:14 +0300 Subject: [PATCH 30/34] =?UTF-8?q?feat(triage):=20clearing=20gate=20?= =?UTF-8?q?=E2=80=94=20weak=20judge=20tiers=20can=20detect+flag=20but=20ne?= =?UTF-8?q?ver=20auto-dismiss=20(ADR-0052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Structural no-false-clear guarantee for weak/thin-lineup configs. The flash run showed that on a weak judge the deep dive can both phantom-clear a real vuln and have all disproof reviewers miss the bypass -> a false-clear (the one verdict that can HIDE a real vuln). Production never hits this for known providers (resolve_tier_model_ids always derives a strong judge: opus/gpt-5/pro), but a thin-lineup provider (ollama/custom/unrecognized) collapses every tier to one possibly-weak model. Fix: clearing_is_trusted(model_full_id) is True only for known-lineup providers. DeepDiveRunner(can_clear=...) gates a single point: any DISMISSAL verdict (unexploitable/false_positive) on an untrusted config is routed to needs_review with a 'Tier gate' note. Detection (real) and flagging (needs_review) stay unrestricted on every tier. Wired into both construction sites (integration + the live eval pipeline). 58 tests green incl. the gate branches. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/runtime/model_tiers.py | 16 ++++- .../cliff/agents/triage_deep/integration.py | 6 +- backend/cliff/agents/triage_deep/runner.py | 44 ++++++++++--- backend/cliff/evals/runners.py | 6 +- backend/tests/agents/test_deep_dive_runner.py | 62 ++++++++++++++++++- backend/tests/agents/test_model_tiers.py | 15 +++++ 6 files changed, 137 insertions(+), 12 deletions(-) diff --git a/backend/cliff/agents/runtime/model_tiers.py b/backend/cliff/agents/runtime/model_tiers.py index 8524e78f..2cc12844 100644 --- a/backend/cliff/agents/runtime/model_tiers.py +++ b/backend/cliff/agents/runtime/model_tiers.py @@ -51,4 +51,18 @@ def judge_is_independent(model_full_id: str) -> bool: return ids["judge"] != ids["strong"] -__all__ = ["TIERS", "judge_is_independent", "resolve_tier_model_ids"] +def clearing_is_trusted(model_full_id: str) -> bool: + """True when the config has a known strong-judge lineup, so the Deep dive may + emit a DISMISSAL verdict (``unexploitable`` / ``false_positive``). + + Clearing is the only verdict that can HIDE a real vuln, so it requires a + capable judge tier. A thin-lineup config (ollama / custom / unrecognized id) + collapses every tier to one possibly-weak model — there the deep dive may + DETECT (``real``) and FLAG (``needs_review``) on any tier, but must never + auto-dismiss (the flash-judge false-clears were exactly this weak-judge + failure). Known-lineup providers always derive a strong judge (opus / gpt-5 / + pro), so dismissal is trusted.""" + return model_full_id.partition("/")[0] in _LINEUP + + +__all__ = ["TIERS", "clearing_is_trusted", "judge_is_independent", "resolve_tier_model_ids"] diff --git a/backend/cliff/agents/triage_deep/integration.py b/backend/cliff/agents/triage_deep/integration.py index 9e463358..aeacb11d 100644 --- a/backend/cliff/agents/triage_deep/integration.py +++ b/backend/cliff/agents/triage_deep/integration.py @@ -12,6 +12,7 @@ import logging from typing import TYPE_CHECKING, Any +from cliff.agents.runtime.model_tiers import clearing_is_trusted from cliff.agents.triage_deep.escalation import ( DEFAULT_DEEP_DIVE_BUDGET, decide_escalation, @@ -76,7 +77,10 @@ async def maybe_deep_dive( name: mgr.read_artifact(repo.id, name) for name in ("profile", "code_map", "threat") } - active_runner = runner or DeepDiveRunner(build_tier_models(ai_env, model_full_id)) + active_runner = runner or DeepDiveRunner( + build_tier_models(ai_env, model_full_id), + can_clear=clearing_is_trusted(model_full_id), + ) return await active_runner.run( finding=finding, repo_knowledge=knowledge, diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index 75276242..bfee498e 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -134,9 +134,37 @@ def __init__( models: dict[str, Any], *, stages: DeepDiveStages | None = None, + can_clear: bool = True, ) -> None: self._models = models self._stages = stages or DeepDiveStages() + # Safety net: when the configured judge tier isn't a known-capable model + # (thin-lineup ollama/custom), the Deep dive may detect + flag but must + # never auto-dismiss — every clear verdict is routed to needs_review. + self._can_clear = can_clear + + _CLEAR_VERDICTS = ("unexploitable", "false_positive") + + def _gate_clear(self, output: TriageOutput) -> TriageOutput: + """Downgrade a DISMISSAL to needs_review on a config whose judge tier can't + be trusted to clear (structural no-false-clear guarantee for weak tiers).""" + if self._can_clear or output.verdict not in self._CLEAR_VERDICTS: + return output + return output.model_copy( + update={ + "verdict": "needs_review", + "checks": [ + *(output.checks or []), + TriageCheck( + eyebrow="Tier gate", + result="Analysis cleared this finding, but the configured " + "model tier is below the auto-dismiss threshold — routed to " + "review rather than dismissed.", + kind="warn", + ), + ], + } + ) async def run( self, @@ -159,13 +187,15 @@ def incomplete(reason: str) -> TriageOutput: ) try: - return await self._run( - finding=finding, - repo_knowledge=repo_knowledge, - clone_dir=clone_dir, - enrichment=enrichment, - exposure=exposure, - traced_sha=traced_sha, + return self._gate_clear( + await self._run( + finding=finding, + repo_knowledge=repo_knowledge, + clone_dir=clone_dir, + enrichment=enrichment, + exposure=exposure, + traced_sha=traced_sha, + ) ) except UsageLimitExceeded: return incomplete("Analysis hit the request budget") diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index c5ada843..97a3d222 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -433,9 +433,13 @@ def make_live_deep_dive_pipeline( The case's ``finding`` may carry ``repo_knowledge`` / ``enrichment`` / ``exposure`` that the staged pipeline reads. """ + from cliff.agents.runtime.model_tiers import clearing_is_trusted from cliff.agents.triage_deep.runner import DeepDiveRunner, build_tier_models - runner = DeepDiveRunner(build_tier_models(env, model_full_id)) + runner = DeepDiveRunner( + build_tier_models(env, model_full_id), + can_clear=clearing_is_trusted(model_full_id), + ) async def _run(case: EvalCase, repo_dir: Path) -> Any: finding = case.finding diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py index e99beac4..45bdc446 100644 --- a/backend/tests/agents/test_deep_dive_runner.py +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -34,8 +34,8 @@ async def _f(deps, model): return _f -async def _run(stages, rk=RK): - runner = DeepDiveRunner(MODELS, stages=stages) +async def _run(stages, rk=RK, *, can_clear=True): + runner = DeepDiveRunner(MODELS, stages=stages, can_clear=can_clear) return await runner.run( finding={"title": "x"}, repo_knowledge=rk, clone_dir="/tmp/x", traced_sha="sha1" ) @@ -218,3 +218,61 @@ async def test_challenge_downgrade_lowers_verdict(): out = await _run(stages) assert out.verdict == "needs_review" assert out.challenge.verdict_holds is False + + +# ── safety net: a weak/thin-lineup judge tier may detect + flag, never clear ── + + +async def test_weak_tier_disproof_clear_downgraded_to_needs_review(): + # can_clear=False: a would-be `unexploitable` (disproof upheld) -> needs_review. + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage( + {"reached": "no", "disproof": {"guard_location": "a.py:1", "explanation": "ok"}} + ), + disproof_challenge=_disproof( + Challenge( + verdict_holds=True, + reviewers=[ChallengeReviewer(lens="bypass", verdict="holds")], + ) + ), + ) + out = await _run(stages, can_clear=False) + assert out.verdict == "needs_review" + assert any(c.eyebrow == "Tier gate" for c in out.checks) + + +async def test_weak_tier_rule_out_kill_downgraded_to_needs_review(): + # can_clear=False: even a corroborated false_positive kill -> needs_review. + rk = {"threat": {"prior_issues": [{"id": "GHSA-x"}]}} + stages = DeepDiveStages( + gather=_stage({"vuln_class": "rce"}), + rule_out=_stage( + { + "killed": True, + "kill_class": "duplicate_of_known", + "recommended_verdict_on_kill": "false_positive", + } + ), + ) + out = await _run(stages, rk, can_clear=False) + assert out.verdict == "needs_review" + + +async def test_weak_tier_still_reports_real(): + # Detection is unaffected — a `real` verdict stays real on a weak tier. + stages = DeepDiveStages( + gather=_stage({}), + rule_out=_stage({"killed": False}), + trace=_stage({"reached": "yes", "path": [{"file": "a.py", "line": 1, "role": "sink"}]}), + plan=_stage({"hypotheses": [{"id": "h1", "trigger_condition": "x"}]}), + challenge=_challenge( + Challenge( + verdict_holds=True, + reviewers=[ChallengeReviewer(lens="exploit", verdict="holds")], + ) + ), + ) + out = await _run(stages, can_clear=False) + assert out.verdict == "real" diff --git a/backend/tests/agents/test_model_tiers.py b/backend/tests/agents/test_model_tiers.py index ad20f4ea..4a0bcdde 100644 --- a/backend/tests/agents/test_model_tiers.py +++ b/backend/tests/agents/test_model_tiers.py @@ -3,6 +3,7 @@ from __future__ import annotations from cliff.agents.runtime.model_tiers import ( + clearing_is_trusted, judge_is_independent, resolve_tier_model_ids, ) @@ -37,3 +38,17 @@ def test_no_slash_falls_back(): def test_judge_independence_signal(): assert judge_is_independent("anthropic/claude-haiku-4-5") is True assert judge_is_independent("ollama/llama3") is False + + +def test_clearing_trusted_for_known_lineups_only(): + # Known providers derive a capable judge → auto-dismiss trusted. + for mid in ( + "anthropic/claude-haiku-4-5", + "openai/gpt-5-mini", + "google/gemini-2.5-flash", + "openrouter/anthropic/claude-haiku-4.5", + ): + assert clearing_is_trusted(mid) is True + # Thin-lineup / unknown → weak judge possible → must NOT auto-dismiss. + for mid in ("ollama/llama3", "custom/my-model", "weird-id"): + assert clearing_is_trusted(mid) is False From 2c3f73e4ab85397209a352458dbdd70b88ecdc9f Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Sat, 13 Jun 2026 18:36:03 +0300 Subject: [PATCH 31/34] =?UTF-8?q?feat(triage):=20Phase=204=20=E2=80=94=20s?= =?UTF-8?q?urface=20the=20Deep=20dive=20in=20the=20triage=20UI=20(ADR-0052?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backend already persists exploit_plan / challenge / provenance to sidebar_state.triage; this surfaces them. Frontend TriageOutput type extended with the deep-dive blocks, and three Cyberdeck components added to SPTriage: - ExploitPlanPanel — ranked exploit hypotheses (impact class, attacker input, reached sink, expected impact) + a collapsible reproduction recipe, framed as 'a plan, not a demonstrated exploit'; the reachable-but-no-exploit case renders a calm hardening note. - ChallengePanel — the adversarial panel: held vs downgraded, per-reviewer lens + holds/refuted + refutation text. - DeepDiveProvenance — 'How Cliff dug in': the stage trail in user-facing names (Gather the facts -> Rule out -> Trace the path -> Plan the exploit -> Challenge the verdict) + the SHA the verdict is valid for. No 1px borders, mono eyebrows, sentence case, --cd-* tiers. tsc + eslint clean, 52 IssueSidePanel tests green (3 new). Live per-stage narration (SSE) is a noted future enhancement (no streaming channel today). Co-Authored-By: Claude Opus 4.8 --- frontend/src/api/client.ts | 60 ++++ .../src/components/issues/IssueSidePanel.tsx | 301 ++++++++++++++++++ .../issues/__tests__/IssueSidePanel.test.tsx | 100 ++++++ 3 files changed, 461 insertions(+) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index d06df4d8..df4bc48e 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -317,6 +317,62 @@ export interface TriageCheck { detail?: string | null; } +// --- Deep dive (ADR-0052) — additive, all optional --- + +export interface TriageReproRecipe { + setup: string[]; + docker_compose?: string | null; + image?: string | null; + ports: number[]; + trigger: string[]; + expected_observation?: string | null; +} + +export interface TriageExploitHypothesis { + id: string; + trigger_condition: string; + attacker_input?: string | null; + /** file:line of the reached sink */ + reached_sink?: string | null; + expected_impact?: string | null; + /** RCE | SSRF | SQLi | … */ + impact_class?: string | null; + repro_recipe?: TriageReproRecipe | null; + confidence: number; +} + +export interface TriageExploitPlan { + hypotheses: TriageExploitHypothesis[]; + primary_hypothesis_id?: string | null; + /** reachable-but-not-exploitable (hardening) signal */ + no_credible_exploit: boolean; +} + +export interface TriageChallengeReviewer { + /** reachability | exploit | impact | bypass | scope | phantom */ + lens: string; + verdict: 'holds' | 'refuted'; + refutation?: string | null; +} + +export interface TriageChallenge { + verdict_holds: boolean; + reviewers: TriageChallengeReviewer[]; + downgraded_verdict?: TriageVerdict | null; + confidence_adjustment: number; +} + +export interface TriageProvenance { + /** stage narration: gather_facts, rule_out, trace_path, plan_exploit, … */ + steps_run: string[]; + /** the commit SHA the verdict is valid for */ + traced_sha?: string | null; + /** step -> tier (cheap | strong | judge) */ + model_tiers: Record; + exit_stage?: string | null; + escalated: boolean; +} + export interface TriageOutput { verdict: TriageVerdict; /** 0.0–1.0; render as word + % (e.g. "High · 92%"), never bare. */ @@ -326,6 +382,10 @@ export interface TriageOutput { exploitability?: TriageExploitability | null; report?: TriageReport | null; checks: TriageCheck[]; + // Deep dive (ADR-0052) — additive, all optional. + exploit_plan?: TriageExploitPlan | null; + challenge?: TriageChallenge | null; + provenance?: TriageProvenance | null; } export interface SidebarState { diff --git a/frontend/src/components/issues/IssueSidePanel.tsx b/frontend/src/components/issues/IssueSidePanel.tsx index 05f99a32..438f3530 100644 --- a/frontend/src/components/issues/IssueSidePanel.tsx +++ b/frontend/src/components/issues/IssueSidePanel.tsx @@ -25,7 +25,9 @@ import type { Finding, IssueStage, TriageCheck, + TriageExploitHypothesis, TriageOutput, + TriageReproRecipe, TriageVerdict, } from '../../api/client' import { @@ -1023,6 +1025,302 @@ function DraftedReply({ triage }: { triage: TriageOutput }) { ) } +// --- Deep dive (ADR-0052): exploit plan, challenge trail, provenance --- + +const DEEP_DIVE_STAGE_LABEL: Record = { + gather_facts: 'Gather the facts', + rule_out: 'Rule out false alarms', + trace_path: 'Trace the path', + plan_exploit: 'Plan the exploit', + challenge: 'Challenge the verdict', + disproof_challenge: 'Challenge the verdict', +} + +function verdictWord(v?: TriageVerdict | null): string { + if (v === 'real') return 'real' + if (v === 'unexploitable') return 'unexploitable' + if (v === 'false_positive') return 'false positive' + return 'needs review' +} + +function RecipeSteps({ label, steps }: { label: string; steps: string[] }) { + return ( +
+ {label} +
    + {steps.map((s, i) => ( +
  1. + {i + 1}. + {s} +
    2. + ))} +
+
+ ) +} + +function ReproRecipe({ recipe }: { recipe?: TriageReproRecipe | null }) { + if (!recipe) return null + const has = + recipe.setup?.length || + recipe.trigger?.length || + recipe.expected_observation || + recipe.image || + recipe.docker_compose + if (!has) return null + return ( +
+ + Reproduction recipe (plan) + +
+ {recipe.image && ( +
+ image: {recipe.image} + {recipe.ports?.length ? ` · ports ${recipe.ports.join(', ')}` : ''} +
+ )} + {recipe.setup?.length > 0 && } + {recipe.trigger?.length > 0 && } + {recipe.expected_observation && ( +
+ Expected +

+ {recipe.expected_observation} +

+
+ )} +
+
+ ) +} + +function ExploitHypothesisCard({ + h, + primary, +}: { + h: TriageExploitHypothesis + primary: boolean +}) { + const conf = Math.round((h.confidence ?? 0) * 100) + return ( +
+
+ {primary && ( + + Primary + + )} + {h.impact_class && ( + {h.impact_class} + )} + {conf > 0 && ( + + {conf}% conf + + )} +
+
+ {h.trigger_condition} +
+ {h.attacker_input && ( +
+ + Attacker input:{' '} + + + {h.attacker_input} + +
+ )} + {h.reached_sink && ( +
+ + Reaches:{' '} + + + {h.reached_sink} + +
+ )} + {h.expected_impact && ( +

+ {h.expected_impact} +

+ )} + +
+ ) +} + +function ExploitPlanPanel({ triage }: { triage: TriageOutput }) { + const plan = triage.exploit_plan + if (!plan) return null + // Reachable but no credible exploit — a calm, positive hardening note. + if (plan.no_credible_exploit) { + return ( +
+ + shield + + + Reachable, but no credible exploit — hardening, not a vulnerability. + +
+ ) + } + if (plan.hypotheses.length === 0) return null + const ordered = [...plan.hypotheses].sort( + (a, b) => + Number(b.id === plan.primary_hypothesis_id) - Number(a.id === plan.primary_hypothesis_id), + ) + return ( +
+ + Exploit plan + {plan.hypotheses.length > 1 ? ` — ${plan.hypotheses.length} hypotheses` : ''} + +

+ A plan, not a demonstrated exploit — Cliff reasons about it, it never runs it. +

+
+ {ordered.map((h) => ( + + ))} +
+
+ ) +} + +function ChallengePanel({ triage }: { triage: TriageOutput }) { + const ch = triage.challenge + if (!ch || ch.reviewers.length === 0) return null + const refuted = ch.reviewers.filter((r) => r.verdict === 'refuted').length + return ( +
+ + Challenge the verdict — {ch.reviewers.length} adversarial reviewer + {ch.reviewers.length === 1 ? '' : 's'} + +
+ + {ch.verdict_holds ? 'verified' : 'change_circle'} + + + {ch.verdict_holds + ? 'Verdict held' + : `Downgraded to ${verdictWord(ch.downgraded_verdict)}`} + + {refuted > 0 && ( + + {refuted} of {ch.reviewers.length} refuted + + )} +
+
    + {ch.reviewers.map((r, i) => { + const kind = r.verdict === 'holds' ? 'pass' : 'warn' + return ( +
  • + + {CHECK_ICON[kind]} + +
    +
    + {r.lens} + + {r.verdict === 'holds' ? 'Holds' : 'Refuted'} + +
    + {r.refutation && ( +

    + {r.refutation} +

    + )} +
    +
    • + ) + })} +
+
+ ) +} + +function DeepDiveProvenance({ triage }: { triage: TriageOutput }) { + const p = triage.provenance + if (!p || !p.escalated) return null // only for findings that went through the Deep dive + const steps = p.steps_run.filter((s) => s !== 'incomplete') + return ( +
+ + How Cliff dug in + + {steps.length > 0 && ( +
+ {steps.map((s, i) => ( + + + {DEEP_DIVE_STAGE_LABEL[s] ?? s} + + {i < steps.length - 1 && ( + + chevron_right + + )} + + ))} +
+ )} + {p.traced_sha && ( +
+ Valid for + + {p.traced_sha.slice(0, 10)} + + + — re-triage if the code changes + +
+ )} +
+ ) +} + function SPTriage({ workspaceId, stage, @@ -1045,9 +1343,12 @@ function SPTriage({ + + + ) : stage === 'triaging' ? ( diff --git a/frontend/src/components/issues/__tests__/IssueSidePanel.test.tsx b/frontend/src/components/issues/__tests__/IssueSidePanel.test.tsx index 06145070..9212f888 100644 --- a/frontend/src/components/issues/__tests__/IssueSidePanel.test.tsx +++ b/frontend/src/components/issues/__tests__/IssueSidePanel.test.tsx @@ -1076,3 +1076,103 @@ describe('IssueSidePanel — triage verdict', () => { ).toBeInTheDocument() }) }) + +// --- Deep dive (ADR-0052 / Phase 4): exploit plan, challenge trail, provenance --- + +const DEEP_DIVE_TRIAGE = { + verdict: 'real', + confidence: 0.9, + recommended_close: null, + reachability: { reached: true, path: [{ label: 'entry', detail: 'app.py:1' }], summary: null }, + exploitability: { exploitable: 'yes', reason: 'reachable with a credible path' }, + report: null, + checks: [], + exploit_plan: { + hypotheses: [ + { + id: 'h1', + trigger_condition: 'GET /file= with a URL-like value', + attacker_input: 'http://169.254.169.254/latest/meta-data/', + reached_sink: 'gradio/routes.py:438', + expected_impact: 'SSRF to the cloud metadata endpoint', + impact_class: 'SSRF', + confidence: 0.8, + repro_recipe: { + setup: ['pip install gradio'], + docker_compose: null, + image: 'gradio:vuln', + ports: [7860], + trigger: ['curl "http://target/file=http://169.254.169.254/"'], + expected_observation: 'metadata document returned', + }, + }, + ], + primary_hypothesis_id: 'h1', + no_credible_exploit: false, + }, + challenge: { + verdict_holds: true, + reviewers: [ + { lens: 'reachability', verdict: 'holds', refutation: null }, + { lens: 'exploit', verdict: 'refuted', refutation: 'input is normalized upstream' }, + ], + downgraded_verdict: null, + confidence_adjustment: 0, + }, + provenance: { + steps_run: ['gather_facts', 'rule_out', 'trace_path', 'plan_exploit', 'challenge'], + traced_sha: 'dc131b64f05062447643217819ca630e483a11df', + model_tiers: {}, + exit_stage: 'challenge', + escalated: true, + }, +} + +function mockSidebarWithDeepDive() { + server.use( + http.get('/api/workspaces/:wsId/sidebar', () => + HttpResponse.json({ + workspace_id: 'ws-1', + summary: null, + evidence: null, + owner: null, + plan: null, + definition_of_done: null, + linked_ticket: null, + validation: null, + similar_cases: null, + pull_request: null, + triage: DEEP_DIVE_TRIAGE, + updated_at: '2026-06-13T00:00:00Z', + }), + ), + ) +} + +describe('IssueSidePanel — Deep dive UI (Phase 4)', () => { + it('renders the exploit plan with its hypothesis + repro recipe', async () => { + mockSidebarWithDeepDive() + renderPanel(findingForStage('triage_verdict')) + expect(await screen.findByText(/Exploit plan/)).toBeInTheDocument() + expect(screen.getByText('SSRF')).toBeInTheDocument() + expect(screen.getByText(/GET \/file= with a URL-like value/)).toBeInTheDocument() + expect(screen.getByText('gradio/routes.py:438')).toBeInTheDocument() + expect(screen.getByText(/Reproduction recipe \(plan\)/)).toBeInTheDocument() + }) + + it('renders the challenge panel with reviewer verdicts', async () => { + mockSidebarWithDeepDive() + renderPanel(findingForStage('triage_verdict')) + expect(await screen.findByText(/adversarial reviewer/)).toBeInTheDocument() + expect(screen.getByText('Verdict held')).toBeInTheDocument() + expect(screen.getByText('input is normalized upstream')).toBeInTheDocument() + }) + + it('renders the provenance trail with friendly stage names + traced sha', async () => { + mockSidebarWithDeepDive() + renderPanel(findingForStage('triage_verdict')) + expect(await screen.findByText('How Cliff dug in')).toBeInTheDocument() + expect(screen.getByText('Trace the path')).toBeInTheDocument() + expect(screen.getByText('dc131b64f0')).toBeInTheDocument() + }) +}) From 46b0f2ff8786c99af7e9eebe5c300f5aed7430a6 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Sun, 14 Jun 2026 17:56:24 +0300 Subject: [PATCH 32/34] =?UTF-8?q?fix(triage):=20code-review=20pass=20?= =?UTF-8?q?=E2=80=94=20close=202=20false-clear=20vectors=20+=20a=20groundi?= =?UTF-8?q?ng-gate=20hole=20(ADR-0052/0054)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review of the branch surfaced real defects in the no-false-clear path: P0 — _gate_clear left a stale recommended_close (unexploitable/false_positive) after downgrading a clear to needs_review (model_copy skips the validator), so the UI pre-selected the dismissal the tier gate just refused. Now sets None + re-tones surviving 'pass' checks to info. P1 — _kill_corroborated false-clear vectors at the cheap gate: * root_cause_in_nonship_code used any() — one stray test file alongside real ship-code cleared the finding. Now requires ALL candidates non-ship (+ empty guard). * duplicate_of_known cleared on any prior issue in history. Now requires the kill's dedup_match to name a prior issue that actually exists. Eval gate hole — check_citation_grounding skipped checks[].detail without a '/', but the disproof guard_location / rule_out kill_evidence (a CLEAR verdict's load-bearing citation) is a bare file:line (e.g. auth.py:10). A fabricated guard at a nonexistent file passed the HARD grounding gate. Now grounds every detail; line 0 no longer resolves. Eval robustness — run_deep_dive_eval now raises on 0 cases (no silent PASS) and isolates per-case infra failures (one bad checkout no longer aborts the run); the live pipeline threads traced_sha for provenance. Runner now degrades httpx.TransportError to needs_review (never crash). Frontend: render docker_compose in the repro recipe; don't render an empty provenance panel on the incomplete path. New/updated tests for every fix; backend (changed code) + frontend green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/triage_deep/runner.py | 36 ++++++++++- backend/cliff/evals/deep_dive_evaluators.py | 13 ++-- backend/cliff/evals/runners.py | 63 +++++++++++-------- backend/tests/agents/test_deep_dive_runner.py | 37 +++++++++++ backend/tests/agents/test_evals_deep_dive.py | 18 ++++++ .../src/components/issues/IssueSidePanel.tsx | 14 +++++ 6 files changed, 147 insertions(+), 34 deletions(-) diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index bfee498e..b76d5971 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +import httpx from pydantic_ai.exceptions import ModelHTTPError, UsageLimitExceeded from cliff.agents.runtime.deps import WorkspaceDeps @@ -99,11 +100,23 @@ def _kill_corroborated(ro: dict, facts: dict, repo_knowledge: dict) -> bool: """ kill_class = ro.get("kill_class") if kill_class == "duplicate_of_known": - return bool((repo_knowledge.get("threat") or {}).get("prior_issues")) + # The kill must name a SPECIFIC prior issue (dedup_match) that exists in + # the threat history — not merely "the repo has some history". Otherwise a + # hallucinated duplicate clears on any repo with any unrelated prior CVE. + prior_ids = { + pi.get("id") for pi in ((repo_knowledge.get("threat") or {}).get("prior_issues") or []) + } + dedup = ro.get("dedup_match") + return bool(dedup and dedup in prior_ids) if kill_class == "root_cause_in_nonship_code": excluded = (repo_knowledge.get("code_map") or {}).get("excluded_roots") or [] files = [c.get("file", "") for c in (facts.get("root_cause_candidates") or [])] - return any(fnmatch.fnmatch(f, pat) for f in files for pat in excluded) + # EVERY candidate must be non-ship (the intent is "root cause lives ONLY in + # non-shipping code"). One stray test file alongside real ship-code must NOT + # clear the finding. Empty candidate list is not corroboration. + return bool(files) and all( + any(fnmatch.fnmatch(f, pat) for pat in excluded) for f in files + ) return False @@ -150,11 +163,23 @@ def _gate_clear(self, output: TriageOutput) -> TriageOutput: be trusted to clear (structural no-false-clear guarantee for weak tiers).""" if self._can_clear or output.verdict not in self._CLEAR_VERDICTS: return output + # Re-tone surviving "pass" checks (e.g. "Not reachable (challenge upheld)") + # to neutral info — they backed a dismissal we no longer trust, so a green + # pass row under a needs_review verdict would mislead. + retoned = [ + c.model_copy(update={"kind": "info"}) if c.kind == "pass" else c + for c in (output.checks or []) + ] return output.model_copy( update={ "verdict": "needs_review", + # `recommended_close` is a coherent projection of `verdict` + # (schema invariant); model_copy skips the validator, so set the + # needs_review-coherent value (None) explicitly — never leave the + # stale "unexploitable"/"false_positive" the gate just refused. + "recommended_close": None, "checks": [ - *(output.checks or []), + *retoned, TriageCheck( eyebrow="Tier gate", result="Analysis cleared this finding, but the configured " @@ -212,6 +237,11 @@ def incomplete(reason: str) -> TriageOutput: ): return incomplete("Analysis could not complete (provider/context)") raise + except httpx.TransportError: + # A hung/dropped connection that survived the per-agent retries — same + # degrade contract as a transient provider outage (never crash, never a + # false clear). Auth/billing/other errors still surface above. + return incomplete("Analysis could not complete (provider/context)") async def _run( self, diff --git a/backend/cliff/evals/deep_dive_evaluators.py b/backend/cliff/evals/deep_dive_evaluators.py index fb1df7f5..d2b8fe1d 100644 --- a/backend/cliff/evals/deep_dive_evaluators.py +++ b/backend/cliff/evals/deep_dive_evaluators.py @@ -45,9 +45,12 @@ def _iter_citations(triage: dict[str, Any]) -> list[str]: if hyp.get("reached_sink"): cites.append(str(hyp["reached_sink"])) for check in triage.get("checks") or []: - detail = check.get("detail") - if detail and "/" in str(detail): # likely a file path (disproof guard, etc.) - cites.append(str(detail)) + # The disproof guard_location / rule_out kill_evidence (a CLEAR verdict's + # load-bearing citation) lands here as a bare ``file:line`` — often with no + # ``/`` (e.g. ``auth.py:10``). Extract every detail and let the file:line + # regex in check_citation_grounding decide; prose simply won't match. + if check.get("detail"): + cites.append(str(check["detail"])) return cites @@ -74,8 +77,8 @@ def check_citation_grounding(triage: dict[str, Any], repo_dir: Path) -> tuple[bo n_lines = len(target.read_text(errors="replace").splitlines()) except OSError: continue - if int(line) > n_lines: - bad.append(f"{raw} (line {line} > {n_lines})") + if int(line) < 1 or int(line) > n_lines: + bad.append(f"{raw} (line {line} out of range 1..{n_lines})") if bad: return False, "fabricated citation(s): " + "; ".join(bad) return True, "all cited file:line resolve" diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 97a3d222..c46411d4 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -384,36 +384,46 @@ async def run_deep_dive_eval( (every cited file:line resolves), read-only tool boundary. GRADED: verdict match against a floor. """ + if not cases: + raise ValueError( + "run_deep_dive_eval got 0 cases — check the dataset path / tier filter " + "(a silent empty run would falsely report PASS)." + ) result = EvalRunResult(agent="triage_deep_dive", n_cases=len(cases), graded_floor=graded_floor) matches: list[bool] = [] for case in cases: - with tempfile.TemporaryDirectory() as tmp: - repo_dir = Path(tmp) / "repo" - if case.repo and case.sha: - # Live lane: walk the REAL repo at the pinned commit. - from cliff.evals.repo_fetch import checkout_at_sha - - await checkout_at_sha(case.repo, case.sha, repo_dir) - else: - # CI / synthetic: stage the inline micro-repo. - repo_dir.mkdir() - for rel, text in (case.files or {}).items(): - fp = repo_dir / rel - fp.parent.mkdir(parents=True, exist_ok=True) - fp.write_text(text) - - triage = await run_pipeline(case, repo_dir) - golden = case.expected.as_dict().get("verdict") - - ok, reason = check_false_clear(triage.verdict, golden) - if not ok: - result.hard_failures.append(f"{case.id}: {reason}") - ok, reason = check_citation_grounding(triage.model_dump(), repo_dir) - if not ok: - result.hard_failures.append(f"{case.id}: {reason}") - if golden is not None: - matches.append(check_verdict_match(triage.verdict, golden)[0]) + try: + with tempfile.TemporaryDirectory() as tmp: + repo_dir = Path(tmp) / "repo" + if case.repo and case.sha: + # Live lane: walk the REAL repo at the pinned commit. + from cliff.evals.repo_fetch import checkout_at_sha + + await checkout_at_sha(case.repo, case.sha, repo_dir) + else: + # CI / synthetic: stage the inline micro-repo. + repo_dir.mkdir() + for rel, text in (case.files or {}).items(): + fp = repo_dir / rel + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(text) + + triage = await run_pipeline(case, repo_dir) + golden = case.expected.as_dict().get("verdict") + + ok, reason = check_false_clear(triage.verdict, golden) + if not ok: + result.hard_failures.append(f"{case.id}: {reason}") + ok, reason = check_citation_grounding(triage.model_dump(), repo_dir) + if not ok: + result.hard_failures.append(f"{case.id}: {reason}") + if golden is not None: + matches.append(check_verdict_match(triage.verdict, golden)[0]) + except Exception as exc: # noqa: BLE001 — one case's infra failure (e.g. a + # checkout error / network blip) must not abort the whole ship gate; + # record it and continue scoring the rest. + result.hard_failures.append(f"{case.id}: infra error — {type(exc).__name__}: {exc}") ok, reason = check_tool_boundary() if not ok: @@ -449,6 +459,7 @@ async def _run(case: EvalCase, repo_dir: Path) -> Any: clone_dir=repo_dir, enrichment=finding.get("enrichment"), exposure=finding.get("exposure"), + traced_sha=case.sha, # provenance: the commit this verdict is valid for ) return _run diff --git a/backend/tests/agents/test_deep_dive_runner.py b/backend/tests/agents/test_deep_dive_runner.py index 45bdc446..9d3025c9 100644 --- a/backend/tests/agents/test_deep_dive_runner.py +++ b/backend/tests/agents/test_deep_dive_runner.py @@ -50,6 +50,7 @@ async def test_corroborated_kill_exits_at_rule_out(): { "killed": True, "kill_class": "duplicate_of_known", + "dedup_match": "GHSA-x", # names the specific prior issue "recommended_verdict_on_kill": "false_positive", } ), @@ -60,6 +61,39 @@ async def test_corroborated_kill_exits_at_rule_out(): assert out.provenance.steps_run == ["gather_facts", "rule_out"] +async def test_duplicate_kill_without_matching_id_falls_through(): + # A duplicate_of_known kill that doesn't name a prior issue present in the + # threat history is NOT corroborated — it must fall through to trace, never + # clear on "the repo has some history". + rk = {"threat": {"prior_issues": [{"id": "GHSA-x"}]}} + stages = DeepDiveStages( + gather=_stage({"vuln_class": "rce"}), + rule_out=_stage( + {"killed": True, "kill_class": "duplicate_of_known", "dedup_match": "GHSA-unrelated"} + ), + trace=_stage({"reached": "unknown"}), + ) + out = await _run(stages, rk) + assert out.verdict == "needs_review" # fell through, did not clear at rule_out + assert "trace_path" in out.provenance.steps_run + + +async def test_nonship_kill_needs_all_candidates_nonship(): + # root_cause_in_nonship_code must hold for EVERY candidate; one ship-code + # candidate alongside a test file must NOT clear at the cheap gate. + rk = {"code_map": {"excluded_roots": ["tests/*"]}} + stages = DeepDiveStages( + gather=_stage( + {"root_cause_candidates": [{"file": "tests/test_x.py"}, {"file": "app/views.py"}]} + ), + rule_out=_stage({"killed": True, "kill_class": "root_cause_in_nonship_code"}), + trace=_stage({"reached": "unknown"}), + ) + out = await _run(stages, rk) + assert out.verdict == "needs_review" # fell through (app/views.py is ship code) + assert "trace_path" in out.provenance.steps_run + + async def test_uncorroborated_kill_falls_through_to_trace(): # A "looks safe" kill with no structural backing must NOT clear at the cheap # gate — it falls through to trace_path (the false-clear guarantee). @@ -252,12 +286,15 @@ async def test_weak_tier_rule_out_kill_downgraded_to_needs_review(): { "killed": True, "kill_class": "duplicate_of_known", + "dedup_match": "GHSA-x", "recommended_verdict_on_kill": "false_positive", } ), ) out = await _run(stages, rk, can_clear=False) assert out.verdict == "needs_review" + # the gate must also clear the stale recommended_close, not leave 'false_positive' + assert out.recommended_close is None async def test_weak_tier_still_reports_real(): diff --git a/backend/tests/agents/test_evals_deep_dive.py b/backend/tests/agents/test_evals_deep_dive.py index 23691970..3b6e8b90 100644 --- a/backend/tests/agents/test_evals_deep_dive.py +++ b/backend/tests/agents/test_evals_deep_dive.py @@ -57,6 +57,24 @@ def test_citation_grounding_skips_prose(tmp_path): assert check_citation_grounding(prose, tmp_path)[0] is True +def test_citation_grounding_catches_fabricated_disproof_guard(tmp_path): + # A CLEAR verdict's load-bearing citation (the disproof guard / rule_out + # evidence) lands in checks[].detail as a bare file:line with no '/'. The gate + # must still ground it — a fabricated guard at a nonexistent file fails. + (tmp_path / "app.py").write_text("line1\nline2\n") + upheld = {"verdict": "unexploitable", "checks": [{"eyebrow": "Disproof", "detail": "app.py:1"}]} + assert check_citation_grounding(upheld, tmp_path)[0] is True + fabricated = { + "verdict": "unexploitable", + "checks": [{"eyebrow": "Disproof", "detail": "ghost.py:99"}], + } + assert check_citation_grounding(fabricated, tmp_path)[0] is False + # line 0 doesn't resolve either + assert check_citation_grounding( + {"checks": [{"detail": "app.py:0"}]}, tmp_path + )[0] is False + + def test_tool_boundary_is_read_only(): ok, reason = check_tool_boundary() assert ok is True diff --git a/frontend/src/components/issues/IssueSidePanel.tsx b/frontend/src/components/issues/IssueSidePanel.tsx index 438f3530..0dd588c3 100644 --- a/frontend/src/components/issues/IssueSidePanel.tsx +++ b/frontend/src/components/issues/IssueSidePanel.tsx @@ -1083,6 +1083,17 @@ function ReproRecipe({ recipe }: { recipe?: TriageReproRecipe | null }) { {recipe.ports?.length ? ` · ports ${recipe.ports.join(', ')}` : ''} )} + {recipe.docker_compose && ( +
+ docker-compose + 
+              {recipe.docker_compose}
+
+
+ )} {recipe.setup?.length > 0 && } {recipe.trigger?.length > 0 && } {recipe.expected_observation && ( @@ -1278,6 +1289,9 @@ function DeepDiveProvenance({ triage }: { triage: TriageOutput }) { const p = triage.provenance if (!p || !p.escalated) return null // only for findings that went through the Deep dive const steps = p.steps_run.filter((s) => s !== 'incomplete') + // The incomplete-degrade path sets escalated=true but has no steps + no SHA — + // don't render an empty "How Cliff dug in" disclosure. + if (steps.length === 0 && !p.traced_sha) return null return (
Date: Sun, 14 Jun 2026 18:02:23 +0300 Subject: [PATCH 33/34] polish(triage): keep sage-mint for safe outcomes only in the deep-dive UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cyberdeck /impeccable polish pass: green is the safe/status-good accent, used sparingly. Removed it from two spots where it was decorative or semantically backwards on a real finding — the 'Primary' exploit-hypothesis badge (the main attack path is not 'safe' → neutral fg-2, first position carries the emphasis) and the 'Verdict held' challenge icon (confirmatory, not safe → neutral fg-3; amber still flags a downgrade). 'Primary' now only labels when there's >1 hypothesis to rank (redundant on a single one). tsc + eslint + 52 tests green. Co-Authored-By: Claude Opus 4.8 --- .../src/components/issues/IssueSidePanel.tsx | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/issues/IssueSidePanel.tsx b/frontend/src/components/issues/IssueSidePanel.tsx index 0dd588c3..2118e116 100644 --- a/frontend/src/components/issues/IssueSidePanel.tsx +++ b/frontend/src/components/issues/IssueSidePanel.tsx @@ -1121,7 +1121,10 @@ function ExploitHypothesisCard({
{primary && ( - + // Neutral, not green: green is the "safe / status-good" accent, and the + // primary attack path on a real finding is the opposite of safe. fg-2 + + // first position carry the emphasis calmly. + Primary )} @@ -1188,6 +1191,9 @@ function ExploitPlanPanel({ triage }: { triage: TriageOutput }) { ) } if (plan.hypotheses.length === 0) return null + // "Primary" only earns its label when there's more than one hypothesis to rank; + // on a single hypothesis it's redundant noise. + const markPrimary = plan.hypotheses.length > 1 const ordered = [...plan.hypotheses].sort( (a, b) => Number(b.id === plan.primary_hypothesis_id) - Number(a.id === plan.primary_hypothesis_id), @@ -1206,7 +1212,11 @@ function ExploitPlanPanel({ triage }: { triage: TriageOutput }) {

{ordered.map((h) => ( - + ))}
@@ -1230,8 +1240,10 @@ function ChallengePanel({ triage }: { triage: TriageOutput }) { Date: Sun, 14 Jun 2026 18:16:19 +0300 Subject: [PATCH 34/34] fix(triage): address CodeRabbit + Baz review (ADR-0052/0054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bot review of #268. The two highest findings (the _kill_corroborated dedup_match corroboration + the empty-dataset guard) were already closed by the earlier self-review pass. This batch: - ReadBudget.take(): reject an over-budget request instead of letting the first one go negative — the off-by-one let one oversized read overflow the context (CodeRabbit, deps.py). - clearing_is_trusted(): reject malformed ids (no '/') — only a real provider/model config can be trusted to auto-dismiss (CodeRabbit, model_tiers). - citation grounding: constrain resolution to repo_dir (a '..'/absolute citation can no longer pass by matching a host file) + line 0 is out of range (CodeRabbit, deep_dive_evaluators). - EvalCase: enforce the dataset-mode invariant (repo+sha together, not alongside files) so a partial case can't silently fall back to synthetic (CodeRabbit). - repos profile route: InvalidRepoUrlError -> 422, not 500 (CodeRabbit). - _run_scanner_triage: finding typed Finding, not Any (CodeRabbit, coding guide). - report path now routes through maybe_deep_dive(source='report') so reports can escalate to the Deep dive like scanner findings, best-effort (Baz). Tests updated for the corrected ReadBudget semantics + new tests for the citation repo-escape and the EvalCase invariant. Affected backend tests + lint green. Co-Authored-By: Claude Opus 4.8 --- backend/cliff/agents/runtime/deps.py | 8 ++-- backend/cliff/agents/runtime/model_tiers.py | 5 ++- backend/cliff/agents/triage_runner.py | 45 ++++++++++++++++++-- backend/cliff/api/routes/repos.py | 7 ++- backend/cliff/evals/cases.py | 13 +++++- backend/cliff/evals/deep_dive_evaluators.py | 8 ++++ backend/tests/agents/test_evals_deep_dive.py | 24 +++++++++++ backend/tests/agents/test_read_budget.py | 23 +++++++--- 8 files changed, 117 insertions(+), 16 deletions(-) diff --git a/backend/cliff/agents/runtime/deps.py b/backend/cliff/agents/runtime/deps.py index a94fe16e..cc074736 100644 --- a/backend/cliff/agents/runtime/deps.py +++ b/backend/cliff/agents/runtime/deps.py @@ -25,9 +25,11 @@ class ReadBudget: remaining: int # bytes def take(self, n: int) -> bool: - """Reserve *n* bytes. Returns False once the budget is spent (the caller - then returns a short 'budget exhausted' marker instead of more content).""" - if self.remaining <= 0: + """Reserve *n* bytes. Returns False when the request would exceed the cap + (the caller then returns a short 'budget exhausted' marker instead of more + content). Rejecting an over-budget request — rather than allowing the first + one to go negative — is what actually prevents the context-window overflow.""" + if n > self.remaining: return False self.remaining -= n return True diff --git a/backend/cliff/agents/runtime/model_tiers.py b/backend/cliff/agents/runtime/model_tiers.py index 2cc12844..e3ec540e 100644 --- a/backend/cliff/agents/runtime/model_tiers.py +++ b/backend/cliff/agents/runtime/model_tiers.py @@ -62,7 +62,10 @@ def clearing_is_trusted(model_full_id: str) -> bool: auto-dismiss (the flash-judge false-clears were exactly this weak-judge failure). Known-lineup providers always derive a strong judge (opus / gpt-5 / pro), so dismissal is trusted.""" - return model_full_id.partition("/")[0] in _LINEUP + provider, sep, model = model_full_id.partition("/") + # A malformed id (no '/', e.g. "openai") isn't a valid provider/model config — + # never trust dismissal on it. + return bool(sep and model) and provider in _LINEUP __all__ = ["TIERS", "clearing_is_trusted", "judge_is_independent", "resolve_tier_model_ids"] diff --git a/backend/cliff/agents/triage_runner.py b/backend/cliff/agents/triage_runner.py index e0d8c5d0..40e2b4d9 100644 --- a/backend/cliff/agents/triage_runner.py +++ b/backend/cliff/agents/triage_runner.py @@ -33,7 +33,7 @@ if TYPE_CHECKING: import aiosqlite - from cliff.models import Workspace + from cliff.models import Finding, Workspace logger = logging.getLogger(__name__) @@ -126,7 +126,15 @@ async def run_triage( raise ValueError(f"workspace {workspace.id} has no finding") if finding.source_type == REPORT_SOURCE_TYPE: - return await _run_report_triage(executor, db, workspace, env_vars=env_vars) + return await _run_report_triage( + executor, + db, + workspace, + finding=finding, + env_vars=env_vars, + ai_env=ai_env, + model_full_id=model_full_id, + ) return await _run_scanner_triage( executor, db, @@ -143,7 +151,7 @@ async def _run_scanner_triage( db: aiosqlite.Connection, workspace: Workspace, *, - finding: Any, + finding: Finding, env_vars: dict[str, str], ai_env: dict[str, str] | None = None, model_full_id: str | None = None, @@ -206,7 +214,10 @@ async def _run_report_triage( db: aiosqlite.Connection, workspace: Workspace, *, + finding: Finding, env_vars: dict[str, str], + ai_env: dict[str, str] | None = None, + model_full_id: str | None = None, ) -> TriageOutput | None: """Run the report triager (read-only repo access). It persists its own chat card + ``sidebar.triage`` via the executor's standard path; we read @@ -228,7 +239,33 @@ async def _run_report_triage( sidebar = await get_sidebar(db, workspace.id) if sidebar is None or not sidebar.triage: return None - return TriageOutput.model_validate(sidebar.triage) + quick = TriageOutput.model_validate(sidebar.triage) + + # Escalate to the agentic Deep dive when warranted (ADR-0052 — the report + # variant starts at gather_facts). Best-effort: any failure keeps the report + # triager's verdict — triage never breaks. + final = quick + try: + deep = await maybe_deep_dive( + db, + finding={**finding.model_dump(mode="json")}, + quick=quick, + repo_url=workspace.repo_url, + enrichment=None, + exposure=None, + ai_env=ai_env, + model_full_id=model_full_id, + source="report", + ) + if deep is not None: + final = deep + await _persist_synthesis(db, workspace.id, final) + except Exception: + logger.exception( + "deep dive failed for report workspace %s — keeping the quick verdict", + workspace.id, + ) + return final __all__ = ["REPORT_SOURCE_TYPE", "SYNTHESIZER_AGENT_TYPE", "run_triage"] diff --git a/backend/cliff/api/routes/repos.py b/backend/cliff/api/routes/repos.py index 1f162ab3..b21478f3 100644 --- a/backend/cliff/api/routes/repos.py +++ b/backend/cliff/api/routes/repos.py @@ -17,6 +17,7 @@ from cliff.db.connection import get_db from cliff.repos.dao import get_repo_by_url +from cliff.repos.identity import InvalidRepoUrlError from cliff.repos.service import schedule_profile_build router = APIRouter(prefix="/repos", tags=["repos"]) @@ -71,7 +72,11 @@ async def get_profile( if url is None: return RepoProfileStatus() - repo = await get_repo_by_url(db, url) + try: + repo = await get_repo_by_url(db, url) + except InvalidRepoUrlError as exc: + # A malformed repo_url is a client error, not a 500. + raise HTTPException(status_code=422, detail=str(exc)) from exc if repo is None: return RepoProfileStatus(repo_url=url, status="none") diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index 4a17d995..33fd38d6 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator # Pydantic resolves ``Expected.verdict``'s annotation at model-build time, so # this is a genuine runtime import (not type-only) despite ``from __future__ @@ -84,6 +84,17 @@ class EvalCase(BaseModel): repo: str | None = None sha: str | None = None + @model_validator(mode="after") + def _dataset_mode_invariant(self) -> EvalCase: + """``repo``+``sha`` (live lane) and ``files`` (synthetic) are mutually + exclusive, and ``repo``/``sha`` must come as a pair — otherwise the runner + silently falls back to synthetic staging and the result is meaningless.""" + if (self.repo is None) != (self.sha is None): + raise ValueError("repo and sha must be provided together") + if self.repo is not None and self.files is not None: + raise ValueError("a case is real (repo+sha) or synthetic (files), not both") + return self + def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: """Load ``.jsonl`` from the active dataset dir into typed cases.""" diff --git a/backend/cliff/evals/deep_dive_evaluators.py b/backend/cliff/evals/deep_dive_evaluators.py index d2b8fe1d..6397e13d 100644 --- a/backend/cliff/evals/deep_dive_evaluators.py +++ b/backend/cliff/evals/deep_dive_evaluators.py @@ -69,6 +69,14 @@ def check_citation_grounding(triage: dict[str, Any], repo_dir: Path) -> tuple[bo continue rel, line = m.group(1), m.group(2) target = repo / rel + # A citation must point INSIDE the staged repo. An absolute path or `..` + # traversal that escapes repo_dir is a fabrication, not a resolving file — + # never let it pass the gate by matching something on the host. + try: + target.resolve().relative_to(repo.resolve()) + except ValueError: + bad.append(f"{raw} (escapes repo)") + continue if not target.is_file(): bad.append(f"{raw} (file not found)") continue diff --git a/backend/tests/agents/test_evals_deep_dive.py b/backend/tests/agents/test_evals_deep_dive.py index 3b6e8b90..846a3596 100644 --- a/backend/tests/agents/test_evals_deep_dive.py +++ b/backend/tests/agents/test_evals_deep_dive.py @@ -8,6 +8,8 @@ from __future__ import annotations +import pytest + from cliff.agents.schemas import ( ExploitHypothesis, ExploitPlan, @@ -75,6 +77,28 @@ def test_citation_grounding_catches_fabricated_disproof_guard(tmp_path): )[0] is False +def test_citation_grounding_rejects_repo_escape(tmp_path): + # A `..`/absolute citation that escapes the staged repo is a fabrication — + # it must not pass the gate by matching a file on the host. + (tmp_path / "app.py").write_text("line1\n") + escape = {"reachability": {"path": [{"detail": "../../../outside.py:1"}]}} + assert check_citation_grounding(escape, tmp_path)[0] is False + + +def test_eval_case_dataset_mode_invariant(): + # synthetic (files) and live (repo+sha) are both valid on their own. + EvalCase(id="syn", finding={}, files={"a.py": "x"}) + EvalCase(id="live", finding={}, repo="https://github.com/o/r", sha="abc123") + # repo without sha (or vice versa) → silent synthetic fallback; reject it. + with pytest.raises(ValueError): + EvalCase(id="partial", finding={}, repo="https://github.com/o/r") + # both modes at once is ambiguous → reject. + with pytest.raises(ValueError): + EvalCase( + id="both", finding={}, files={"a.py": "x"}, repo="https://github.com/o/r", sha="abc" + ) + + def test_tool_boundary_is_read_only(): ok, reason = check_tool_boundary() assert ok is True diff --git a/backend/tests/agents/test_read_budget.py b/backend/tests/agents/test_read_budget.py index be8189cb..65f241be 100644 --- a/backend/tests/agents/test_read_budget.py +++ b/backend/tests/agents/test_read_budget.py @@ -14,9 +14,11 @@ def test_read_budget_take(): b = ReadBudget(10) assert b.take(7) is True assert b.remaining == 3 - assert b.take(5) is True # allowed (was > 0), may overshoot - assert b.remaining == -2 - assert b.take(1) is False # now exhausted + assert b.take(5) is False # would exceed the cap → refused, budget unchanged + assert b.remaining == 3 + assert b.take(3) is True # exact fit + assert b.remaining == 0 + assert b.take(1) is False # exhausted def _ctx(tmp_path, budget): @@ -28,13 +30,22 @@ def _ctx(tmp_path, budget): async def test_read_stops_once_budget_spent(tmp_path): (tmp_path / "f.txt").write_text("x" * 100) - ctx = _ctx(tmp_path, ReadBudget(50)) + ctx = _ctx(tmp_path, ReadBudget(100)) # fits exactly one read first = await read(ctx, "f.txt") - assert first.startswith("x") # first read goes through (budget was > 0) + assert first.startswith("x") # first read fits the budget second = await read(ctx, "f.txt") assert "budget exhausted" in second # budget now spent → refused +async def test_read_refuses_a_single_over_budget_file(tmp_path): + # The fix: a file larger than the remaining budget is refused outright, not + # returned once — returning it would overflow the context (the original bug). + (tmp_path / "big.txt").write_text("x" * 100) + ctx = _ctx(tmp_path, ReadBudget(50)) # smaller than the file + out = await read(ctx, "big.txt") + assert "budget exhausted" in out + + async def test_no_budget_is_unlimited(tmp_path): (tmp_path / "f.txt").write_text("y" * 100) ctx = _ctx(tmp_path, None) # the executor's case — unchanged behaviour @@ -45,7 +56,7 @@ async def test_no_budget_is_unlimited(tmp_path): async def test_grep_respects_budget(tmp_path): (tmp_path / "a.py").write_text("needle here\n") - ctx = _ctx(tmp_path, ReadBudget(5)) + ctx = _ctx(tmp_path, ReadBudget(30)) # fits one match line (~19 bytes), not two first = await grep(ctx, "needle") assert "a.py" in first # first grep returns matches second = await grep(ctx, "needle")