From 6653101b7ad1be0b3d4c805cda1e51f5e19b6985 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Mon, 8 Jun 2026 23:54:41 +0300 Subject: [PATCH 1/7] feat(evals): agent eval harness + finding_enricher reference impl (ADR-0050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First agent wired under the generic eval system (ADR-0050 rollout: highest- risk first). Thin layer over pydantic-evals: - cliff/evals/: adapter (drives any workspace agent through one call), registry (AgentEvalSpec — supported assertions + budget + model), cases (typed JSONL loader), evaluators (deterministic; "code first, judge last"). - The citation gate is SPLIT (data-driven, the live run taught us this): structural fabrication (fake GHSA / garbled SHA / non-http) is the zero-tolerance HARD gate (reuses the production reference_verifier); a plausible URL that 404s is a GRADED dead-link metric, not a hard fail. - Two lanes (ADR-0050 §5): test_evals_harness.py (CI, deterministic, FunctionModel/TestModel + injected http — proves the scorer); test_evals_finding_enricher.py (live, key-gated via conftest — runs the real enricher over eval/finding_enricher.jsonl: hard gates zero-tolerance, graded metrics on a floor). - Dataset: 6 cases covering well-known CVE, dependency CVE, scanner-jargon title, two no-CVE abstention cases, and a fabrication-bait case. Verified: CI lane 9/9 + 104 deterministic agent tests green; the live lane passes against real gpt-4o-mini (no structural fabrication, abstention holds, graded metrics clear the floor). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/__init__.py | 21 ++ backend/cliff/evals/adapter.py | 53 +++++ backend/cliff/evals/cases.py | 55 +++++ backend/cliff/evals/evaluators.py | 214 ++++++++++++++++++ backend/cliff/evals/registry.py | 82 +++++++ backend/tests/agents/conftest.py | 6 +- .../tests/agents/eval/finding_enricher.jsonl | 6 + .../agents/test_evals_finding_enricher.py | 88 +++++++ backend/tests/agents/test_evals_harness.py | 144 ++++++++++++ 9 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 backend/cliff/evals/__init__.py create mode 100644 backend/cliff/evals/adapter.py create mode 100644 backend/cliff/evals/cases.py create mode 100644 backend/cliff/evals/evaluators.py create mode 100644 backend/cliff/evals/registry.py create mode 100644 backend/tests/agents/eval/finding_enricher.jsonl create mode 100644 backend/tests/agents/test_evals_finding_enricher.py create mode 100644 backend/tests/agents/test_evals_harness.py diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py new file mode 100644 index 00000000..dc73e5ca --- /dev/null +++ b/backend/cliff/evals/__init__.py @@ -0,0 +1,21 @@ +"""Cliff agent-evaluation harness (ADR-0050). + +A thin, generic layer over ``pydantic-evals``: a per-agent registry, an +adapter that drives any agent through one call, and a small set of custom +evaluators. Datasets live as JSONL under ``backend/tests/agents/eval/``. + +Two lanes (ADR-0050 §5): + +* **CI** — deterministic, ``FunctionModel``/``TestModel``, every push. Proves + the evaluators + adapter are correct without a key. +* **Live** — key-gated, real model, measures actual agent quality. + +The first agent wired is ``finding_enricher`` (the reference implementation, +ADR-0050 rollout §7). +""" + +from cliff.evals.adapter import run_agent +from cliff.evals.cases import EvalCase, load_cases +from cliff.evals.registry import AgentEvalSpec, get_spec + +__all__ = ["AgentEvalSpec", "EvalCase", "get_spec", "load_cases", "run_agent"] diff --git a/backend/cliff/evals/adapter.py b/backend/cliff/evals/adapter.py new file mode 100644 index 00000000..c6c95974 --- /dev/null +++ b/backend/cliff/evals/adapter.py @@ -0,0 +1,53 @@ +"""Generic agent adapter for the eval harness (ADR-0050 §1). + +One call drives any workspace-scoped runtime agent: build the model, build +the agent, construct ``WorkspaceDeps`` from the case input, render the same +user prompt the executor uses, and run. The model can be injected (a +``FunctionModel`` for the deterministic CI lane) or built from canonical AI +state (the live lane). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from cliff.agents.runtime._prompts import build_user_prompt +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.provider import build_model + +if TYPE_CHECKING: + from pydantic_ai.models import Model + + from cliff.evals.registry import AgentEvalSpec + + +async def run_agent( + spec: AgentEvalSpec, + finding: dict[str, Any], + *, + env: dict[str, str] | None = None, + model_id: str | None = None, + model: Model | None = None, + prior_context: dict[str, dict[str, Any]] | None = None, +) -> Any: + """Run *spec*'s agent over a single eval case and return its output object. + + Provide ``model`` directly (CI lane: a ``FunctionModel``/``TestModel``) or + ``env`` + ``model_id`` to build a real model (live lane). The returned + object is the agent's structured ``output_type`` instance (e.g. + ``EnrichmentOutput``), exactly what evaluators score. + """ + resolved_model = model if model is not None else build_model(env or {}, model_id) + agent = spec.build_agent(resolved_model) + deps = WorkspaceDeps( + workspace_id="eval", + workspace_dir="/tmp/cliff-eval", + finding=finding, + prior_context=prior_context or {}, + env_vars=env or {}, + ) + result = await agent.run(build_user_prompt(deps), deps=deps) + return result.output + + +__all__ = ["run_agent"] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py new file mode 100644 index 00000000..d8b1306f --- /dev/null +++ b/backend/cliff/evals/cases.py @@ -0,0 +1,55 @@ +"""Eval dataset cases (ADR-0050 §2). + +One typed schema for every agent's cases, stored as JSONL at +``backend/tests/agents/eval/.jsonl`` — one case per line, append a +line to add a case. ``load_cases`` enumerates them in file order. + +(The dataset lives under ``tests/`` and is only read by the eval tests; the +loader resolving a ``tests/`` path from a ``cliff.*`` module is the +test/prod-line blur tracked as ADR-0050 Open question #7.) +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, Field + +# backend/ root → backend/cliff/evals/cases.py is parents[2] +_EVAL_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval" + + +class EvalCase(BaseModel): + """One eval case. ``finding`` is the raw input; ``expected`` holds the + golden labels the deterministic evaluators check; ``abstain`` marks a + case where the agent MUST decline (no CVE / post-cutoff).""" + + id: str + tier: Literal["ci", "live"] = "live" + edge_case: str | None = None + abstain: bool = False + finding: dict[str, Any] + expected: dict[str, Any] = Field(default_factory=dict) + + +def load_cases(agent: str, *, tier: str | None = None) -> list[EvalCase]: + """Load ``.jsonl`` into typed cases, optionally filtered by tier.""" + path = _EVAL_DIR / f"{agent}.jsonl" + if not path.is_file(): + raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}") + cases: list[EvalCase] = [] + for line_no, raw in enumerate(path.read_text().splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("//"): + continue + try: + cases.append(EvalCase.model_validate_json(line)) + except ValueError as exc: # malformed line — surface which one + raise ValueError(f"{path.name}:{line_no}: invalid case — {exc}") from exc + if tier is not None: + cases = [c for c in cases if c.tier == tier] + return cases + + +__all__ = ["EvalCase", "load_cases"] diff --git a/backend/cliff/evals/evaluators.py b/backend/cliff/evals/evaluators.py new file mode 100644 index 00000000..d6b97bc8 --- /dev/null +++ b/backend/cliff/evals/evaluators.py @@ -0,0 +1,214 @@ +"""Evaluators for the finding_enricher (ADR-0050 §3). + +"Code first, judge last." Everything here is deterministic. The headline gate +is ``citation_liveness``: it reuses the production reference verifier +(``services.reference_verifier.clean_references``) — if that verifier would +*drop* any reference the agent emitted (structurally bogus GHSA/SHA, or a URL +the host 404s), the agent fabricated a citation and the case FAILS. Production +already strips these before the user sees them; the eval measures how often the +model tries. + +Each check is a pure ``(passed: bool, reason: str)`` function (clear pytest +messages, trivially unit-tested). The ``pydantic-evals`` ``Evaluator`` wrappers +at the bottom call them so the Dataset/Report path (ADR-0050 §1) works too. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from cliff.services.reference_verifier import clean_references + +# Scanner-specific jargon that a normalized title must not carry (ADR-0040: +# "a human should understand the title without knowing which scanner produced +# it"). Raw rule ids, leading bracket tags, dotted semgrep rule paths. +_JARGON_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bSNYK-[A-Z0-9-]+", re.IGNORECASE), + re.compile(r"\bGHSA-[a-z0-9]{4}-", re.IGNORECASE), + re.compile(r"^\s*\[[^\]]+\]"), # leading "[semgrep] ..." style tag + re.compile(r"\b[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_.-]+\b"), # dotted rule path +) + + +def _norm_cves(values: Any) -> set[str]: + if not isinstance(values, (list, tuple)): + return set() + return {str(v).strip().upper() for v in values if str(v).strip()} + + +@dataclass +class ReferenceAssessment: + """Classification of the verifier's dropped references. + + ``structural`` = structurally-impossible (fabricated GHSA id, garbled + commit SHA, non-http, unsafe IP) — pure fabrication, caught in the + no-network sanitize pass. ``network`` = a plausible URL the host 404/410s + — a dead-link guess. They are scored differently (ADR-0050 §3): structural + is the zero-tolerance hard gate; network is a graded liveness metric. + """ + + structural: list[tuple[str, str]] + network: list[tuple[str, str]] + kept: list[str] + + +async def assess_references(output: Any, *, http: Any = None) -> ReferenceAssessment: + """Run the production verifier once and classify its drops. + + ``http`` (an ``httpx.AsyncClient``) is injectable so the CI lane can drive + the network pass deterministically; the live lane leaves it ``None``. + """ + refs = getattr(output, "references", None) + check = ( + await clean_references(refs, http=http) + if http is not None + else await clean_references(refs) + ) + structural = [(u, r) for u, r in check.dropped if not r.startswith("http_")] + network = [(u, r) for u, r in check.dropped if r.startswith("http_")] + return ReferenceAssessment(structural, network, check.kept) + + +async def check_structural_citations(output: Any, *, http: Any = None) -> tuple[bool, str]: + """HARD GATE: FAIL on a structurally-impossible (fabricated) reference.""" + a = await assess_references(output, http=http) + if a.structural: + bad = "; ".join(f"{u} ({why})" for u, why in a.structural) + return False, f"fabricated references: {bad}" + return True, f"{len(a.kept)} ok, {len(a.network)} dead-link(s)" + + +async def check_reference_liveness(output: Any, *, http: Any = None) -> tuple[bool, str]: + """GRADED: FAIL if any reference 404/410s (a plausible-but-dead URL guess — + production strips these, so this measures the guess rate, not user impact).""" + a = await assess_references(output, http=http) + if a.network: + dead = "; ".join(u for u, _ in a.network) + return False, f"dead links: {dead}" + return True, f"all {len(a.kept)} reference(s) live" + + +def check_cve_ids(output: Any, expected: dict[str, Any]) -> tuple[bool, str]: + """Exact set match against the golden CVE ids (case-insensitive). No + ``cve_ids`` key declared → no expectation → pass (the abstention gate, not + this one, enforces emptiness on no-CVE cases).""" + if "cve_ids" not in expected: + return True, "no cve_ids expectation" + got = _norm_cves(getattr(output, "cve_ids", None)) + want = _norm_cves(expected.get("cve_ids", [])) + if got == want: + return True, f"cve_ids match ({sorted(want) or 'none'})" + return False, f"cve_ids mismatch: got {sorted(got)}, expected {sorted(want)}" + + +def check_cvss_within( + output: Any, expected: dict[str, Any], *, tol: float = 0.5 +) -> tuple[bool, str]: + """CVSS within ±tol of golden, or inside [cvss_min, cvss_max], or null + when the case expects abstention. No expectation declared → pass.""" + got = getattr(output, "cvss_score", None) + if "cvss_score" in expected: + want = expected["cvss_score"] + if want is None: + return (got is None), f"cvss expected null, got {got}" + if got is None: + return False, f"cvss expected ~{want}, got null" + ok = abs(float(got) - float(want)) <= tol + return ok, f"cvss {got} vs {want} (±{tol})" + if "cvss_min" in expected or "cvss_max" in expected: + lo = expected.get("cvss_min", 0.0) + hi = expected.get("cvss_max", 10.0) + if got is None: + return False, f"cvss expected in [{lo},{hi}], got null" + return (lo <= float(got) <= hi), f"cvss {got} in [{lo},{hi}]" + return True, "no cvss expectation" + + +def check_no_jargon_title(output: Any) -> tuple[bool, str]: + """normalized_title must carry no scanner jargon.""" + title = getattr(output, "normalized_title", None) + if not isinstance(title, str) or not title.strip(): + return False, "normalized_title missing/empty" + for pat in _JARGON_PATTERNS: + m = pat.search(title) + if m: + return False, f"title contains scanner jargon {m.group(0)!r}: {title!r}" + return True, "title clean" + + +def check_abstention(output: Any) -> tuple[bool, str]: + """For a no-CVE / post-cutoff case: no invented CVE id and no invented + CVSS score (the agent must decline rather than fabricate).""" + cves = _norm_cves(getattr(output, "cve_ids", None)) + cvss = getattr(output, "cvss_score", None) + if cves: + return False, f"should abstain but invented cve_ids {sorted(cves)}" + if cvss is not None: + return False, f"should abstain but invented cvss_score {cvss}" + return True, "abstained (no invented CVE/CVSS)" + + +# --- pydantic-evals Evaluator wrappers (Dataset/Report path) ----------------- + +try: # pydantic-evals is an optional [evals] extra; degrade if absent + from pydantic_evals.evaluators import Evaluator, EvaluatorContext + + @dataclass + class StructuralCitations(Evaluator): + """Hard gate — no fabricated (structurally-impossible) references.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + passed, _ = await check_structural_citations(ctx.output) + return passed + + @dataclass + class ReferenceLiveness(Evaluator): + """Graded — no dead-link (404/410) references.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + passed, _ = await check_reference_liveness(ctx.output) + return passed + + @dataclass + class CveIds(Evaluator): + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_cve_ids(ctx.output, ctx.expected_output or {})[0] + + @dataclass + class CvssWithin(Evaluator): + tol: float = 0.5 + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_cvss_within(ctx.output, ctx.expected_output or {}, tol=self.tol)[0] + + @dataclass + class NoJargonTitle(Evaluator): + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_no_jargon_title(ctx.output)[0] + + @dataclass + class Abstention(Evaluator): + """Only meaningful on cases flagged ``abstain`` in metadata.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + if not (ctx.metadata or {}).get("abstain"): + return True + return check_abstention(ctx.output)[0] + + _HAS_PYDANTIC_EVALS = True +except ImportError: # pragma: no cover + _HAS_PYDANTIC_EVALS = False + + +__all__ = [ + "ReferenceAssessment", + "assess_references", + "check_abstention", + "check_cve_ids", + "check_cvss_within", + "check_no_jargon_title", + "check_reference_liveness", + "check_structural_citations", +] diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py new file mode 100644 index 00000000..38782722 --- /dev/null +++ b/backend/cliff/evals/registry.py @@ -0,0 +1,82 @@ +"""Per-agent eval registry (ADR-0050 §1). + +One ``AgentEvalSpec`` per eval target — the single source of truth for what +each agent supports, its budget, and how to build it. The runner validates a +dataset's declared assertions against ``supported_assertions``. + +Only ``finding_enricher`` is wired today (ADR-0050 rollout §7: highest-risk +first). Add entries as each agent's eval lands. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from cliff.agents.runtime.finding_enricher import build_agent as _build_enricher +from cliff.agents.schemas import EnrichmentOutput + +if TYPE_CHECKING: + from collections.abc import Callable + + from pydantic import BaseModel + from pydantic_ai import Agent + from pydantic_ai.models import Model + + +@dataclass(frozen=True) +class BudgetSpec: + """Per-case ceilings, enforced by the runner (ADR-0050 §4).""" + + max_usd: float | None = None + max_tokens: int | None = None + max_duration_s: float | None = None + + +@dataclass(frozen=True) +class AgentEvalSpec: + name: str + build_agent: Callable[[Model], Agent] + output_type: type[BaseModel] + abstention_required: bool + supported_assertions: frozenset[str] + budget: BudgetSpec + default_model: str | None = None + live_only: bool = False + eval_frozen: bool = False # deprecated agents (owner_resolver): keep, don't maintain + + +_REGISTRY: dict[str, AgentEvalSpec] = { + "finding_enricher": AgentEvalSpec( + name="finding_enricher", + build_agent=_build_enricher, + output_type=EnrichmentOutput, + abstention_required=True, + supported_assertions=frozenset( + { + "citation_liveness", + "cve_ids", + "cvss_within", + "no_jargon_title", + "abstention", + } + ), + budget=BudgetSpec(max_usd=0.03, max_tokens=8000, max_duration_s=60.0), + ), +} + + +def get_spec(name: str) -> AgentEvalSpec: + try: + return _REGISTRY[name] + except KeyError: + raise KeyError( + f"No eval spec for {name!r}. Registered: {sorted(_REGISTRY)}" + ) from None + + +def all_specs() -> list[AgentEvalSpec]: + return list(_REGISTRY.values()) + + +__all__ = ["AgentEvalSpec", "BudgetSpec", "all_specs", "get_spec"] diff --git a/backend/tests/agents/conftest.py b/backend/tests/agents/conftest.py index 00a496b5..85546c50 100644 --- a/backend/tests/agents/conftest.py +++ b/backend/tests/agents/conftest.py @@ -22,7 +22,11 @@ # Files whose every test hits a real LLM (the live evals). Everything else # under tests/agents/ runs against FunctionModel/TestModel and stays in CI. -_LIVE_LLM_FILES = ("test_normalizer_agent", "test_plain_description_eval") +_LIVE_LLM_FILES = ( + "test_normalizer_agent", + "test_plain_description_eval", + "test_evals_finding_enricher", +) def pytest_collection_modifyitems(items): diff --git a/backend/tests/agents/eval/finding_enricher.jsonl b/backend/tests/agents/eval/finding_enricher.jsonl new file mode 100644 index 00000000..a6612457 --- /dev/null +++ b/backend/tests/agents/eval/finding_enricher.jsonl @@ -0,0 +1,6 @@ +{"id": "log4shell", "tier": "live", "edge_case": "well_known_cve", "abstain": false, "finding": {"title": "Remote code execution via JNDI lookup in Apache Log4j", "description": "The application bundles Apache Log4j 2.14.1. A crafted log message containing a ${jndi:ldap://...} lookup string allows remote code execution.", "raw_severity": "critical", "source_type": "trivy", "source_id": "CVE-2021-44228", "asset_label": "api-service"}, "expected": {"cve_ids": ["CVE-2021-44228"], "cvss_min": 9.0}} +{"id": "lodash-prototype-pollution", "tier": "live", "edge_case": "dependency_cve", "abstain": false, "finding": {"title": "Prototype Pollution in lodash", "description": "lodash 4.17.20 is vulnerable to prototype pollution via the set and setWith functions when used with attacker-controlled property paths.", "raw_severity": "high", "source_type": "snyk", "source_id": "SNYK-JS-LODASH-1018905", "asset_label": "web-frontend"}, "expected": {"cve_ids": ["CVE-2020-8203"], "cvss_min": 5.5, "cvss_max": 8.5}} +{"id": "jargon-title-strip", "tier": "live", "edge_case": "scanner_jargon_title", "abstain": false, "finding": {"title": "SNYK-JS-MINIMIST-559764: Prototype Pollution", "description": "minimist before 1.2.3 is vulnerable to prototype pollution. The title carries the raw Snyk rule id which must be stripped.", "raw_severity": "medium", "source_type": "snyk", "source_id": "SNYK-JS-MINIMIST-559764", "asset_label": "cli-tool"}, "expected": {"cvss_min": 4.0, "cvss_max": 9.9}} +{"id": "semgrep-no-cve-eval", "tier": "live", "edge_case": "no_cve_code_finding", "abstain": true, "finding": {"title": "Use of eval() on user-controlled input", "description": "The handler passes request.body directly to eval(), allowing arbitrary code execution. This is a first-party code defect found by static analysis; it has no CVE.", "raw_severity": "high", "source_type": "semgrep", "source_id": "javascript.lang.security.detect-eval-with-expression", "asset_label": "orders-api"}, "expected": {"cve_ids": []}} +{"id": "posture-no-cve-missing-header", "tier": "live", "edge_case": "no_cve_posture_finding", "abstain": true, "finding": {"title": "Missing Content-Security-Policy header", "description": "Responses from the public site do not set a Content-Security-Policy header. A configuration/posture issue, not a tracked CVE.", "raw_severity": "low", "source_type": "posture", "source_id": "missing_security_header", "asset_label": "marketing-site"}, "expected": {"cve_ids": []}} +{"id": "obscure-pkg-fabrication-bait", "tier": "live", "edge_case": "fabrication_bait", "abstain": true, "finding": {"title": "Vulnerability in internal-telemetry-shim 0.3.1", "description": "A dependency scan flagged internal-telemetry-shim@0.3.1 (a private/obscure package) as potentially vulnerable, but provided no advisory id, CVE, or details. The enricher must not invent a GHSA id, CVE, CVSS, or advisory URL.", "raw_severity": "medium", "source_type": "snyk", "source_id": "", "asset_label": "telemetry"}, "expected": {"cve_ids": []}} diff --git a/backend/tests/agents/test_evals_finding_enricher.py b/backend/tests/agents/test_evals_finding_enricher.py new file mode 100644 index 00000000..5f24e8fd --- /dev/null +++ b/backend/tests/agents/test_evals_finding_enricher.py @@ -0,0 +1,88 @@ +"""Live lane for the finding_enricher eval (ADR-0050 §5, Lane 2). + +Runs the REAL enricher over ``eval/finding_enricher.jsonl`` and scores the +output. Skipped automatically when no API key is set (see ``conftest.py`` — +this file is in ``_LIVE_LLM_FILES``); never masks the deterministic CI lane. + +Two classes of check (ADR-0050 §3): + +* **Hard gates — zero tolerance.** ``citation_liveness`` (real network HEAD via + the production verifier) on every case, and ``abstention`` on every case + flagged ``abstain``. A single failure fails the run: a fabricated citation + or an invented CVE/CVSS is the load-bearing failure mode. +* **Graded — aggregate threshold.** ``cve_ids`` / ``cvss_within`` / + ``no_jargon_title`` are accuracy metrics; we assert the dataset-wide pass + rate clears a floor rather than demanding every case (reasonable models + defensibly miss a CVE id now and then). + +Budget: ~6 LLM calls + a few HTTP HEADs, well under the registry's per-case +ceiling. Tune ``_GRADED_FLOOR`` as the golden set grows. +""" + +from __future__ import annotations + +from cliff.evals import get_spec, load_cases, run_agent +from cliff.evals.evaluators import ( + assess_references, + check_abstention, + check_cve_ids, + check_cvss_within, + check_no_jargon_title, +) +from tests.agents.eval_utils import LLM_ENV as _LLM_ENV +from tests.agents.eval_utils import LLM_MODEL as _LLM_MODEL + +_GRADED_FLOOR = 0.6 # dataset-wide pass-rate floor for accuracy metrics + + +async def test_finding_enricher_eval(): + spec = get_spec("finding_enricher") + cases = load_cases("finding_enricher", tier="live") + assert cases, "no live cases in finding_enricher.jsonl" + + hard_failures: list[str] = [] + graded: dict[str, list[bool]] = { + "cve_ids": [], + "cvss_within": [], + "no_jargon_title": [], + "reference_liveness": [], + } + + for case in cases: + out = await run_agent( + spec, case.finding, env=_LLM_ENV, model_id=_LLM_MODEL + ) + + # One verifier pass per case → classify structural vs dead-link drops. + refs = await assess_references(out) + + # Hard gate (zero tolerance): no STRUCTURALLY-fabricated citation + # (fake GHSA id / garbled SHA / non-http). Dead-link 404s are graded + # below, not a hard fail (production strips them; even good models + # occasionally guess a doc URL that has moved). + if refs.structural: + bad = "; ".join(f"{u} ({why})" for u, why in refs.structural) + hard_failures.append(f"{case.id}: structural_citations — {bad}") + + # Hard gate: declines on no-CVE / fabrication-bait cases. + if case.abstain: + ab_ok, ab_reason = check_abstention(out) + if not ab_ok: + hard_failures.append(f"{case.id}: abstention — {ab_reason}") + + # Graded accuracy metrics. + graded["cve_ids"].append(check_cve_ids(out, case.expected)[0]) + graded["cvss_within"].append(check_cvss_within(out, case.expected)[0]) + graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) + graded["reference_liveness"].append(not refs.network) + + assert not hard_failures, "Zero-tolerance hard-gate failures:\n" + "\n".join( + hard_failures + ) + + below: list[str] = [] + for metric, results in graded.items(): + rate = sum(results) / len(results) + if rate < _GRADED_FLOOR: + below.append(f"{metric}: {rate:.0%} < {_GRADED_FLOOR:.0%}") + assert not below, "Graded metric(s) below floor:\n" + "\n".join(below) diff --git a/backend/tests/agents/test_evals_harness.py b/backend/tests/agents/test_evals_harness.py new file mode 100644 index 00000000..554c5ceb --- /dev/null +++ b/backend/tests/agents/test_evals_harness.py @@ -0,0 +1,144 @@ +"""CI lane for the eval harness (ADR-0050 §5, Lane 1). + +Deterministic, no API key, no real network. Proves the finding_enricher +evaluators + adapter + registry/dataset wiring are correct — so a green CI +means the harness itself won't silently mis-score the live lane. +""" + +from __future__ import annotations + +from pydantic_ai.models.test import TestModel + +from cliff.agents.schemas import EnrichmentOutput +from cliff.evals import get_spec, load_cases, run_agent +from cliff.evals.evaluators import ( + check_abstention, + check_cve_ids, + check_cvss_within, + check_no_jargon_title, + check_reference_liveness, + check_structural_citations, +) + +# --- citation_liveness (the fail-closed gate) -------------------------------- + + +class _FakeResp: + def __init__(self, status_code: int) -> None: + self.status_code = status_code + + +class _FakeHttp: + """Stand-in httpx.AsyncClient: every probe returns *status*.""" + + def __init__(self, status: int = 200) -> None: + self._status = status + + async def get(self, url: str, **_kw: object) -> _FakeResp: + return _FakeResp(self._status) + + async def aclose(self) -> None: # pragma: no cover - never owns the client + pass + + +async def test_structural_gate_fails_on_fabricated_github_sha(): + """A garbled commit SHA is structurally bogus → dropped in the no-network + sanitize pass → the HARD gate FAILS (the agent fabricated a citation).""" + out = EnrichmentOutput( + normalized_title="x", + references=["https://github.com/foo/bar/commit/zzzzzzz"], + ) + passed, reason = await check_structural_citations(out) + assert passed is False + assert "fabricated" in reason + + +async def test_liveness_graded_separates_dead_link_from_fabrication(): + """A plausible URL that 404s is a GRADED dead-link, NOT a structural-gate + failure — the key distinction the live eval taught us.""" + out = EnrichmentOutput( + normalized_title="x", references=["https://example.com/moved-page"] + ) + # structural gate passes (the URL is well-formed)... + structural_ok, _ = await check_structural_citations(out, http=_FakeHttp(404)) + assert structural_ok is True + # ...but the graded liveness check flags the 404. + live_ok, reason = await check_reference_liveness(out, http=_FakeHttp(404)) + assert live_ok is False + assert "dead links" in reason + + +async def test_both_gates_pass_for_live_and_empty(): + live = EnrichmentOutput( + normalized_title="x", + references=["https://nvd.nist.gov/vuln/detail/CVE-2021-44228"], + ) + assert (await check_structural_citations(live, http=_FakeHttp(200)))[0] is True + assert (await check_reference_liveness(live, http=_FakeHttp(200)))[0] is True + empty = EnrichmentOutput(normalized_title="x", references=[]) + assert (await check_structural_citations(empty))[0] is True + assert (await check_reference_liveness(empty))[0] is True + + +# --- deterministic field evaluators ------------------------------------------ + + +def test_cve_ids_set_match_case_insensitive(): + out = EnrichmentOutput(normalized_title="x", cve_ids=["cve-2021-44228"]) + assert check_cve_ids(out, {"cve_ids": ["CVE-2021-44228"]})[0] is True + assert check_cve_ids(out, {"cve_ids": ["CVE-2020-8203"]})[0] is False + + +def test_cvss_within_tolerance_and_range_and_null(): + out = EnrichmentOutput(normalized_title="x", cvss_score=9.8) + assert check_cvss_within(out, {"cvss_score": 10.0})[0] is True # within ±0.5 + assert check_cvss_within(out, {"cvss_score": 7.0})[0] is False + assert check_cvss_within(out, {"cvss_min": 9.0, "cvss_max": 10.0})[0] is True + null_out = EnrichmentOutput(normalized_title="x", cvss_score=None) + assert check_cvss_within(null_out, {"cvss_score": None})[0] is True + + +def test_no_jargon_title_flags_scanner_ids(): + clean = EnrichmentOutput(normalized_title="Prototype pollution in lodash") + assert check_no_jargon_title(clean)[0] is True + jargon = EnrichmentOutput( + normalized_title="SNYK-JS-LODASH-1018905: Prototype Pollution" + ) + assert check_no_jargon_title(jargon)[0] is False + + +def test_abstention_rejects_invented_cve_or_cvss(): + abstained = EnrichmentOutput(normalized_title="x", cve_ids=[], cvss_score=None) + assert check_abstention(abstained)[0] is True + invented_cve = EnrichmentOutput(normalized_title="x", cve_ids=["CVE-2026-99999"]) + assert check_abstention(invented_cve)[0] is False + invented_cvss = EnrichmentOutput(normalized_title="x", cvss_score=7.5) + assert check_abstention(invented_cvss)[0] is False + + +# --- adapter + registry + dataset wiring ------------------------------------- + + +async def test_adapter_runs_enricher_via_testmodel(): + """run_agent builds + drives the real enricher agent; TestModel synthesises + a schema-valid EnrichmentOutput so the wiring is exercised without a key.""" + spec = get_spec("finding_enricher") + out = await run_agent(spec, {"title": "x", "description": "y"}, model=TestModel()) + assert isinstance(out, EnrichmentOutput) + + +def test_dataset_loads_and_assertions_are_supported(): + spec = get_spec("finding_enricher") + cases = load_cases("finding_enricher") + assert len(cases) >= 5 + assert {c.id for c in cases} >= {"log4shell", "semgrep-no-cve-eval"} + # every abstain case declares no expected cve_ids (the abstention contract) + for c in cases: + if c.abstain: + assert c.expected.get("cve_ids", []) == [] + # registry advertises exactly the assertion families this agent's evaluators + # implement (ADR-0050 §1: dataset assertions ⊆ supported_assertions) + assert spec.supported_assertions == frozenset( + {"citation_liveness", "cve_ids", "cvss_within", "no_jargon_title", "abstention"} + ) + assert spec.abstention_required is True From 56d5bdbdb75b655c0a310ba573162f374500cc01 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 11:50:37 +0300 Subject: [PATCH 2/7] =?UTF-8?q?refactor(evals):=20hybrid=20split=20?= =?UTF-8?q?=E2=80=94=20dataset=20dir=20via=20env,=20reusable=20run=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prep for the ADR-0050 hybrid (harness public, data private): - load_cases reads CLIFF_EVAL_DATASET_DIR (default = the public synthetic sample under tests/agents/eval/); the private cliff-os/eval project overrides it to point at the real/confidential golden sets, which never enter this public repo. - Extract the enricher run loop into cliff/evals/runners.run_enricher_eval (+ EvalRunResult) so both the in-repo live test AND the private cliff-os runner drive the same scorer over their own data. - README in tests/agents/eval/ documents the public-sample/private-data split. Verified: CI harness 9/9; live eval passes against gpt-4o-mini on the default sample AND with CLIFF_EVAL_DATASET_DIR pointed at an override dir (the bridge). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/__init__.py | 14 ++- backend/cliff/evals/cases.py | 18 ++- backend/cliff/evals/runners.py | 103 ++++++++++++++++++ backend/tests/agents/eval/README.md | 26 +++++ .../agents/test_evals_finding_enricher.py | 86 +++------------ 5 files changed, 169 insertions(+), 78 deletions(-) create mode 100644 backend/cliff/evals/runners.py create mode 100644 backend/tests/agents/eval/README.md diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py index dc73e5ca..d3907581 100644 --- a/backend/cliff/evals/__init__.py +++ b/backend/cliff/evals/__init__.py @@ -15,7 +15,17 @@ """ from cliff.evals.adapter import run_agent -from cliff.evals.cases import EvalCase, load_cases +from cliff.evals.cases import EvalCase, dataset_dir, load_cases from cliff.evals.registry import AgentEvalSpec, get_spec +from cliff.evals.runners import EvalRunResult, run_enricher_eval -__all__ = ["AgentEvalSpec", "EvalCase", "get_spec", "load_cases", "run_agent"] +__all__ = [ + "AgentEvalSpec", + "EvalCase", + "EvalRunResult", + "dataset_dir", + "get_spec", + "load_cases", + "run_agent", + "run_enricher_eval", +] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index d8b1306f..c8621273 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -11,13 +11,23 @@ from __future__ import annotations +import os from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, Field -# backend/ root → backend/cliff/evals/cases.py is parents[2] -_EVAL_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval" +# Public synthetic sample lives in-repo; backend/ root is parents[2]. +_SAMPLE_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval" + + +def dataset_dir() -> Path: + """Where datasets are read from (ADR-0050 hybrid: harness public, data + private). Defaults to the public synthetic sample; the private eval project + (``cliff-os/eval``) overrides it via ``CLIFF_EVAL_DATASET_DIR`` to point at + the real/confidential golden sets — which never enter this public repo.""" + override = os.environ.get("CLIFF_EVAL_DATASET_DIR") + return Path(override) if override else _SAMPLE_DIR class EvalCase(BaseModel): @@ -34,8 +44,8 @@ class EvalCase(BaseModel): def load_cases(agent: str, *, tier: str | None = None) -> list[EvalCase]: - """Load ``.jsonl`` into typed cases, optionally filtered by tier.""" - path = _EVAL_DIR / f"{agent}.jsonl" + """Load ``.jsonl`` from the active dataset dir into typed cases.""" + path = dataset_dir() / f"{agent}.jsonl" if not path.is_file(): raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}") cases: list[EvalCase] = [] diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py new file mode 100644 index 00000000..1dd3c9e9 --- /dev/null +++ b/backend/cliff/evals/runners.py @@ -0,0 +1,103 @@ +"""Reusable eval run loops (ADR-0050). + +The orchestration lives here — in the public ``cliff`` package — so it can be +driven from BOTH the in-repo live test (public synthetic sample) AND the +private ``cliff-os/eval`` project (real/confidential datasets), which depends +on ``cliff`` and only supplies its own dataset dir. Same scorer, two data +sources (ADR-0050 hybrid). + +Agent-specific for now (only ``finding_enricher`` is wired). Generalize into a +registry-driven loop when the second agent lands. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from cliff.evals.adapter import run_agent +from cliff.evals.evaluators import ( + assess_references, + check_abstention, + check_cve_ids, + check_cvss_within, + check_no_jargon_title, +) +from cliff.evals.registry import get_spec + +if TYPE_CHECKING: + from cliff.evals.cases import EvalCase + + +@dataclass +class EvalRunResult: + agent: str + n_cases: int + graded_floor: float + hard_failures: list[str] = field(default_factory=list) + graded_rates: dict[str, float] = field(default_factory=dict) + + @property + def passed(self) -> bool: + return not self.hard_failures and all( + rate >= self.graded_floor for rate in self.graded_rates.values() + ) + + def report(self) -> str: + lines = [f"{self.agent}: {self.n_cases} cases — {'PASS' if self.passed else 'FAIL'}"] + for metric, rate in sorted(self.graded_rates.items()): + flag = "" if rate >= self.graded_floor else f" ⛔ < {self.graded_floor:.0%}" + lines.append(f" graded {metric:18s} {rate:.0%}{flag}") + for hf in self.hard_failures: + lines.append(f" ⛔ HARD {hf}") + return "\n".join(lines) + + +async def run_enricher_eval( + cases: list[EvalCase], + *, + env: dict[str, str], + model_id: str | None, + graded_floor: float = 0.6, +) -> EvalRunResult: + """Run the real finding_enricher over *cases* and score it. + + Hard gates (zero tolerance, collected into ``hard_failures``): structural + citation fabrication on every case, abstention on every ``abstain`` case. + Graded metrics (dataset-wide pass rate vs ``graded_floor``): cve_ids, + cvss_within, no_jargon_title, reference_liveness. + """ + spec = get_spec("finding_enricher") + result = EvalRunResult(agent=spec.name, n_cases=len(cases), graded_floor=graded_floor) + graded: dict[str, list[bool]] = { + "cve_ids": [], + "cvss_within": [], + "no_jargon_title": [], + "reference_liveness": [], + } + + for case in cases: + out = await run_agent(spec, case.finding, env=env, model_id=model_id) + refs = await assess_references(out) + + if refs.structural: + bad = "; ".join(f"{u} ({why})" for u, why in refs.structural) + result.hard_failures.append(f"{case.id}: structural_citations — {bad}") + if case.abstain: + ok, reason = check_abstention(out) + if not ok: + result.hard_failures.append(f"{case.id}: abstention — {reason}") + + graded["cve_ids"].append(check_cve_ids(out, case.expected)[0]) + graded["cvss_within"].append(check_cvss_within(out, case.expected)[0]) + graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) + graded["reference_liveness"].append(not refs.network) + + result.graded_rates = { + metric: (sum(vals) / len(vals) if vals else 1.0) + for metric, vals in graded.items() + } + return result + + +__all__ = ["EvalRunResult", "run_enricher_eval"] diff --git a/backend/tests/agents/eval/README.md b/backend/tests/agents/eval/README.md new file mode 100644 index 00000000..c374eaa3 --- /dev/null +++ b/backend/tests/agents/eval/README.md @@ -0,0 +1,26 @@ +# Agent eval datasets — public synthetic samples + +These JSONL files are the **public, synthetic sample** datasets for the agent +eval harness (ADR-0050). They exist so the eval is: + +- **runnable in open source** (a contributor can run the live lane), and +- **deterministic in CI** (the harness tests load them). + +They are intentionally small and contain **no proprietary or confidential +data** — only well-known public CVEs and hand-authored synthetic findings. + +## The real datasets live elsewhere (private) + +The proprietary / confidential golden sets — and any customer-derived data — +live in the **private `cliff-os/eval/`** project, never in this public repo +(ADR-0050 "harness public, data private"). + +The harness reads its dataset directory from `CLIFF_EVAL_DATASET_DIR`: + +| Run | `CLIFF_EVAL_DATASET_DIR` | Dataset | +|-----|--------------------------|---------| +| public CI / OSS | _(unset)_ | this directory (synthetic sample) | +| private eval pipeline | `cliff-os/eval/datasets` | the real golden sets | + +So the *scoring logic* is open and the *data* stays private. To add a public +sample case, append a line here; real/sensitive cases go in `cliff-os/eval`. diff --git a/backend/tests/agents/test_evals_finding_enricher.py b/backend/tests/agents/test_evals_finding_enricher.py index 5f24e8fd..1fd4b6c2 100644 --- a/backend/tests/agents/test_evals_finding_enricher.py +++ b/backend/tests/agents/test_evals_finding_enricher.py @@ -1,88 +1,30 @@ """Live lane for the finding_enricher eval (ADR-0050 §5, Lane 2). -Runs the REAL enricher over ``eval/finding_enricher.jsonl`` and scores the -output. Skipped automatically when no API key is set (see ``conftest.py`` — -this file is in ``_LIVE_LLM_FILES``); never masks the deterministic CI lane. +Runs the REAL enricher over the active dataset (the public synthetic sample by +default; the private ``cliff-os/eval`` datasets when ``CLIFF_EVAL_DATASET_DIR`` +is set — ADR-0050 hybrid) and scores it via the shared ``run_enricher_eval`` +loop. Skipped automatically when no API key is set (see ``conftest.py`` — this +file is in ``_LIVE_LLM_FILES``); never masks the deterministic CI lane. -Two classes of check (ADR-0050 §3): +Hard gates (zero tolerance): structural citation fabrication on every case + +abstention on every ``abstain`` case. Graded metrics (cve_ids / cvss_within / +no_jargon_title / reference_liveness) must clear a dataset-wide floor. -* **Hard gates — zero tolerance.** ``citation_liveness`` (real network HEAD via - the production verifier) on every case, and ``abstention`` on every case - flagged ``abstain``. A single failure fails the run: a fabricated citation - or an invented CVE/CVSS is the load-bearing failure mode. -* **Graded — aggregate threshold.** ``cve_ids`` / ``cvss_within`` / - ``no_jargon_title`` are accuracy metrics; we assert the dataset-wide pass - rate clears a floor rather than demanding every case (reasonable models - defensibly miss a CVE id now and then). - -Budget: ~6 LLM calls + a few HTTP HEADs, well under the registry's per-case -ceiling. Tune ``_GRADED_FLOOR`` as the golden set grows. +Budget: a handful of LLM calls + HTTP HEADs, well under the registry ceiling. """ from __future__ import annotations -from cliff.evals import get_spec, load_cases, run_agent -from cliff.evals.evaluators import ( - assess_references, - check_abstention, - check_cve_ids, - check_cvss_within, - check_no_jargon_title, -) +from cliff.evals import load_cases, run_enricher_eval from tests.agents.eval_utils import LLM_ENV as _LLM_ENV from tests.agents.eval_utils import LLM_MODEL as _LLM_MODEL -_GRADED_FLOOR = 0.6 # dataset-wide pass-rate floor for accuracy metrics - async def test_finding_enricher_eval(): - spec = get_spec("finding_enricher") cases = load_cases("finding_enricher", tier="live") - assert cases, "no live cases in finding_enricher.jsonl" - - hard_failures: list[str] = [] - graded: dict[str, list[bool]] = { - "cve_ids": [], - "cvss_within": [], - "no_jargon_title": [], - "reference_liveness": [], - } - - for case in cases: - out = await run_agent( - spec, case.finding, env=_LLM_ENV, model_id=_LLM_MODEL - ) - - # One verifier pass per case → classify structural vs dead-link drops. - refs = await assess_references(out) + assert cases, "no live cases in the active finding_enricher dataset" - # Hard gate (zero tolerance): no STRUCTURALLY-fabricated citation - # (fake GHSA id / garbled SHA / non-http). Dead-link 404s are graded - # below, not a hard fail (production strips them; even good models - # occasionally guess a doc URL that has moved). - if refs.structural: - bad = "; ".join(f"{u} ({why})" for u, why in refs.structural) - hard_failures.append(f"{case.id}: structural_citations — {bad}") - - # Hard gate: declines on no-CVE / fabrication-bait cases. - if case.abstain: - ab_ok, ab_reason = check_abstention(out) - if not ab_ok: - hard_failures.append(f"{case.id}: abstention — {ab_reason}") - - # Graded accuracy metrics. - graded["cve_ids"].append(check_cve_ids(out, case.expected)[0]) - graded["cvss_within"].append(check_cvss_within(out, case.expected)[0]) - graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) - graded["reference_liveness"].append(not refs.network) - - assert not hard_failures, "Zero-tolerance hard-gate failures:\n" + "\n".join( - hard_failures + result = await run_enricher_eval( + cases, env=_LLM_ENV, model_id=_LLM_MODEL, graded_floor=0.6 ) - - below: list[str] = [] - for metric, results in graded.items(): - rate = sum(results) / len(results) - if rate < _GRADED_FLOOR: - below.append(f"{metric}: {rate:.0%} < {_GRADED_FLOOR:.0%}") - assert not below, "Graded metric(s) below floor:\n" + "\n".join(below) + assert result.passed, "\n" + result.report() From 1c0be5fad1362707c24eacb70c5763962d5d364f Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 11:53:02 +0300 Subject: [PATCH 3/7] build(backend): add hatchling build-system so cliff is installable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-eval hybrid (ADR-0050) needs the private cliff-os/eval project to depend on the cliff backend to import the real agents. That requires cliff to be a buildable/installable package — it had no [build-system]. Add hatchling targeting the cliff package. No change to the in-place run flow (uv sync / uv run unchanged); `uv build` now produces cliff--py3-none-any.whl. CI's install-smoke matrix validates the packaging on this PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c94aa4fe..0cd36959 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -49,3 +49,10 @@ markers = [ "docker: install smoke tests that boot the published Docker image (require docker daemon)", "integration: cross-cutting integration tests against an in-memory DB (no external deps, run by default)", ] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["cliff"] From 02411fabb819fed5ba507bed5e22c626a8037c4b Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 12:40:46 +0300 Subject: [PATCH 4/7] feat(evals): enforce per-case/per-run budget + advisory metric calibration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the "declared but not enforced" budget gap (ADR-0050 §4) and calibrates gating against a production-representative run. - run_agent_measured returns token usage + wall-clock; the runner enforces the registry BudgetSpec: per-case max_tokens/max_duration_s (hard) + max_usd (best-effort via pricing.py), and per-run max_run_tokens/max_run_usd (the runaway-bill stop). A breach fails the eval. The report shows total tokens + $ estimate. - reference_liveness is now ADVISORY (reported, non-gating): production strips dead links before the user sees them, so a dead-link guess is a model-quality signal, not user harm. Structural fabrication stays the hard gate. Verified against the production model (claude-haiku-4-5): PASS, ~11k tok / ~$0.022, gating metrics clear the floor; reference_liveness tracked advisory. Notable finding the eval surfaced: haiku guesses dead doc URLs more than gpt-4o-mini (advisory ~33-50%) — a real, customer-relevant model quirk. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/adapter.py | 73 +++++++++++++++++++++++++---- backend/cliff/evals/pricing.py | 40 ++++++++++++++++ backend/cliff/evals/registry.py | 16 ++++++- backend/cliff/evals/runners.py | 83 +++++++++++++++++++++++++++++---- backend/uv.lock | 2 +- 5 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 backend/cliff/evals/pricing.py diff --git a/backend/cliff/evals/adapter.py b/backend/cliff/evals/adapter.py index c6c95974..983ff3e7 100644 --- a/backend/cliff/evals/adapter.py +++ b/backend/cliff/evals/adapter.py @@ -9,6 +9,8 @@ from __future__ import annotations +import time +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from cliff.agents.runtime._prompts import build_user_prompt @@ -21,6 +23,38 @@ from cliff.evals.registry import AgentEvalSpec +@dataclass +class AgentRun: + """A measured single run — the output plus what it cost (ADR-0050 §4).""" + + output: Any + input_tokens: int + output_tokens: int + total_tokens: int + duration_s: float + + +async def _run( + spec: AgentEvalSpec, + finding: dict[str, Any], + *, + env: dict[str, str] | None, + model_id: str | None, + model: Model | None, + prior_context: dict[str, dict[str, Any]] | None, +): + resolved_model = model if model is not None else build_model(env or {}, model_id) + agent = spec.build_agent(resolved_model) + deps = WorkspaceDeps( + workspace_id="eval", + workspace_dir="/tmp/cliff-eval", + finding=finding, + prior_context=prior_context or {}, + env_vars=env or {}, + ) + return await agent.run(build_user_prompt(deps), deps=deps) + + async def run_agent( spec: AgentEvalSpec, finding: dict[str, Any], @@ -37,17 +71,36 @@ async def run_agent( object is the agent's structured ``output_type`` instance (e.g. ``EnrichmentOutput``), exactly what evaluators score. """ - resolved_model = model if model is not None else build_model(env or {}, model_id) - agent = spec.build_agent(resolved_model) - deps = WorkspaceDeps( - workspace_id="eval", - workspace_dir="/tmp/cliff-eval", - finding=finding, - prior_context=prior_context or {}, - env_vars=env or {}, + result = await _run( + spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context ) - result = await agent.run(build_user_prompt(deps), deps=deps) return result.output -__all__ = ["run_agent"] +async def run_agent_measured( + spec: AgentEvalSpec, + finding: dict[str, Any], + *, + env: dict[str, str] | None = None, + model_id: str | None = None, + model: Model | None = None, + prior_context: dict[str, dict[str, Any]] | None = None, +) -> AgentRun: + """Like :func:`run_agent`, but also returns token usage + wall-clock time so + the runner can enforce a per-case / per-run budget.""" + start = time.monotonic() + result = await _run( + spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context + ) + duration = time.monotonic() - start + usage = result.usage + return AgentRun( + output=result.output, + input_tokens=usage.input_tokens or 0, + output_tokens=usage.output_tokens or 0, + total_tokens=usage.total_tokens or 0, + duration_s=duration, + ) + + +__all__ = ["AgentRun", "run_agent", "run_agent_measured"] diff --git a/backend/cliff/evals/pricing.py b/backend/cliff/evals/pricing.py new file mode 100644 index 00000000..f4e168ff --- /dev/null +++ b/backend/cliff/evals/pricing.py @@ -0,0 +1,40 @@ +"""Approximate per-model token pricing for eval budget enforcement (ADR-0050 §4). + +USD per 1M tokens, ``(input, output)`` — provider list prices, approximate. Used +to turn token usage into a ``$`` estimate so the runner can enforce ``max_usd``. +The **token + duration caps are the reliable hard limits**; the ``$`` estimate is +best-effort (returns ``None`` for an unpriced model, and the runner then skips +the ``$`` check rather than guessing). Update as prices change / models are added. +""" + +from __future__ import annotations + +# Match is by substring against the model id (e.g. "openrouter/anthropic/ +# claude-haiku-4.5" matches "claude-haiku-4.5"). +_PRICE_USD_PER_MTOK: dict[str, tuple[float, float]] = { + "claude-haiku-4-5": (1.0, 5.0), + "claude-haiku-4.5": (1.0, 5.0), + "claude-sonnet-4": (3.0, 15.0), + "gpt-4o-mini": (0.15, 0.60), + "gpt-5": (1.25, 10.0), + "gemini-2.5-flash": (0.30, 2.50), +} + + +def _match(model_id: str | None) -> str | None: + m = (model_id or "").lower() + return next((k for k in _PRICE_USD_PER_MTOK if k in m), None) + + +def estimate_cost_usd( + model_id: str | None, input_tokens: int, output_tokens: int +) -> float | None: + """USD estimate for a single run, or ``None`` if the model isn't priced.""" + key = _match(model_id) + if key is None: + return None + in_price, out_price = _PRICE_USD_PER_MTOK[key] + return input_tokens / 1e6 * in_price + output_tokens / 1e6 * out_price + + +__all__ = ["estimate_cost_usd"] diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py index 38782722..c5de1659 100644 --- a/backend/cliff/evals/registry.py +++ b/backend/cliff/evals/registry.py @@ -26,11 +26,17 @@ @dataclass(frozen=True) class BudgetSpec: - """Per-case ceilings, enforced by the runner (ADR-0050 §4).""" + """Budget ceilings, enforced by the runner (ADR-0050 §4). The token + time + caps are reliable hard limits; ``*_usd`` is best-effort (skipped when the + model isn't in the pricing table). A breach fails the eval run.""" + # Per case. max_usd: float | None = None max_tokens: int | None = None max_duration_s: float | None = None + # Per run (whole dataset) — the runaway-bill stop. + max_run_usd: float | None = None + max_run_tokens: int | None = None @dataclass(frozen=True) @@ -61,7 +67,13 @@ class AgentEvalSpec: "abstention", } ), - budget=BudgetSpec(max_usd=0.03, max_tokens=8000, max_duration_s=60.0), + budget=BudgetSpec( + max_usd=0.03, + max_tokens=8000, + max_duration_s=60.0, + max_run_usd=0.50, + max_run_tokens=120_000, + ), ), } diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 1dd3c9e9..bde8eec5 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING -from cliff.evals.adapter import run_agent +from cliff.evals.adapter import run_agent_measured from cliff.evals.evaluators import ( assess_references, check_abstention, @@ -23,6 +23,7 @@ check_cvss_within, check_no_jargon_title, ) +from cliff.evals.pricing import estimate_cost_usd from cliff.evals.registry import get_spec if TYPE_CHECKING: @@ -35,21 +36,43 @@ class EvalRunResult: n_cases: int graded_floor: float hard_failures: list[str] = field(default_factory=list) + budget_failures: list[str] = field(default_factory=list) graded_rates: dict[str, float] = field(default_factory=dict) + # Advisory metrics are reported + tracked but do NOT gate pass/fail — e.g. + # reference_liveness, since production strips dead links so they never reach + # the user. Only structural fabrication (the hard gate) is user-harmful. + advisory: set[str] = field(default_factory=set) + total_tokens: int = 0 + est_cost_usd: float | None = None @property def passed(self) -> bool: - return not self.hard_failures and all( - rate >= self.graded_floor for rate in self.graded_rates.values() + return ( + not self.hard_failures + and not self.budget_failures + and all( + rate >= self.graded_floor + for metric, rate in self.graded_rates.items() + if metric not in self.advisory + ) ) def report(self) -> str: - lines = [f"{self.agent}: {self.n_cases} cases — {'PASS' if self.passed else 'FAIL'}"] + cost = f"~${self.est_cost_usd:.4f}" if self.est_cost_usd is not None else "$?" + lines = [ + f"{self.agent}: {self.n_cases} cases — {'PASS' if self.passed else 'FAIL'}" + f" ({self.total_tokens:,} tok, {cost})" + ] for metric, rate in sorted(self.graded_rates.items()): + if metric in self.advisory: + lines.append(f" advis. {metric:18s} {rate:.0%} (tracked, non-gating)") + continue flag = "" if rate >= self.graded_floor else f" ⛔ < {self.graded_floor:.0%}" lines.append(f" graded {metric:18s} {rate:.0%}{flag}") for hf in self.hard_failures: - lines.append(f" ⛔ HARD {hf}") + lines.append(f" ⛔ HARD {hf}") + for bf in self.budget_failures: + lines.append(f" ⛔ BUDGET {bf}") return "\n".join(lines) @@ -68,18 +91,50 @@ async def run_enricher_eval( cvss_within, no_jargon_title, reference_liveness. """ spec = get_spec("finding_enricher") - result = EvalRunResult(agent=spec.name, n_cases=len(cases), graded_floor=graded_floor) + budget = spec.budget + result = EvalRunResult( + agent=spec.name, + n_cases=len(cases), + graded_floor=graded_floor, + # Dead-link guesses are stripped by production before the user sees + # them, so reference_liveness is a model-quality signal, not a gate. + advisory={"reference_liveness"}, + ) graded: dict[str, list[bool]] = { "cve_ids": [], "cvss_within": [], "no_jargon_title": [], "reference_liveness": [], } + run_cost = 0.0 + run_cost_known = True for case in cases: - out = await run_agent(spec, case.finding, env=env, model_id=model_id) - refs = await assess_references(out) + run = await run_agent_measured(spec, case.finding, env=env, model_id=model_id) + out = run.output + + # Per-case budget (ADR-0050 §4): tokens + duration are hard; $ is + # best-effort (skipped when the model isn't priced). + if budget.max_tokens is not None and run.total_tokens > budget.max_tokens: + result.budget_failures.append( + f"{case.id}: {run.total_tokens:,} tok > {budget.max_tokens:,} cap" + ) + if budget.max_duration_s is not None and run.duration_s > budget.max_duration_s: + result.budget_failures.append( + f"{case.id}: {run.duration_s:.0f}s > {budget.max_duration_s:.0f}s cap" + ) + case_cost = estimate_cost_usd(model_id, run.input_tokens, run.output_tokens) + if case_cost is None: + run_cost_known = False + else: + run_cost += case_cost + if budget.max_usd is not None and case_cost > budget.max_usd: + result.budget_failures.append( + f"{case.id}: ~${case_cost:.4f} > ${budget.max_usd:.4f} cap" + ) + result.total_tokens += run.total_tokens + refs = await assess_references(out) if refs.structural: bad = "; ".join(f"{u} ({why})" for u, why in refs.structural) result.hard_failures.append(f"{case.id}: structural_citations — {bad}") @@ -93,6 +148,18 @@ async def run_enricher_eval( graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) graded["reference_liveness"].append(not refs.network) + # Per-run budget (the runaway-bill stop). + if budget.max_run_tokens is not None and result.total_tokens > budget.max_run_tokens: + result.budget_failures.append( + f"run: {result.total_tokens:,} tok > {budget.max_run_tokens:,} cap" + ) + if run_cost_known: + result.est_cost_usd = run_cost + if budget.max_run_usd is not None and run_cost > budget.max_run_usd: + result.budget_failures.append( + f"run: ~${run_cost:.4f} > ${budget.max_run_usd:.4f} cap" + ) + result.graded_rates = { metric: (sum(vals) / len(vals) if vals else 1.0) for metric, vals in graded.items() diff --git a/backend/uv.lock b/backend/uv.lock index c1d52d57..959f2307 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -503,7 +503,7 @@ wheels = [ [[package]] name = "cliff" version = "0.2.1" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "aiosqlite" }, { name = "cryptography" }, From af806fcf21886c97d4d71e952b2f148c4b8b14db Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 17:42:04 +0300 Subject: [PATCH 5/7] refactor(evals): address PR #265 review (Baz + CodeRabbit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - evaluators: dotted-rule jargon regex no longer matches semantic versions like "2.17.1" (CodeRabbit major); drop the redundant http branch in assess_references (Baz). - cases: `expected` is a typed `Expected` model (extra=forbid) so malformed rows fail in load_cases instead of slipping past the evaluators (Baz); `tier` narrowed to the `Tier` literal (CodeRabbit); `CLIFF_EVAL_DATASET_DIR` is resolved to an absolute path so a relative override can't depend on cwd (Baz). - adapter: honor `spec.default_model` when no model_id (CodeRabbit major); validate output is `spec.output_type` (CodeRabbit); narrow `Any` -> typed Finding/BaseModel boundary (CodeRabbit major). - registry: BudgetSpec + AgentEvalSpec are Pydantic models, frozen (CodeRabbit). - runners: 0 cases now raises instead of silently reporting PASS (Baz); pass the typed expected to evaluators via as_dict(). - pricing: longest-key match so e.g. gpt-4o-mini isn't priced as gpt-4o (Baz). - conftest: widen the live-LLM key gate to all build_model providers (OpenRouter/Gemini/Ollama), not just OpenAI/Anthropic (Baz). Declined: extracting a shared run_agent/run_no_tools_agent helper — they return different shapes (raw output object vs post-processed dict) and the shared plumbing is ~2 lines (simplicity rule). Verified: ruff clean, CI harness 9/9, 104 deterministic agent tests, and the live eval vs claude-haiku-4-5 PASS (the regex fix lifted no_jargon_title to 100% by dropping a semantic-version false positive). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/adapter.py | 48 ++++++++++++++++------ backend/cliff/evals/cases.py | 33 ++++++++++++--- backend/cliff/evals/evaluators.py | 11 +++-- backend/cliff/evals/pricing.py | 5 ++- backend/cliff/evals/registry.py | 30 +++++++------- backend/cliff/evals/runners.py | 11 ++++- backend/tests/agents/conftest.py | 14 +++++-- backend/tests/agents/test_evals_harness.py | 4 +- 8 files changed, 110 insertions(+), 46 deletions(-) diff --git a/backend/cliff/evals/adapter.py b/backend/cliff/evals/adapter.py index 983ff3e7..ecbfe871 100644 --- a/backend/cliff/evals/adapter.py +++ b/backend/cliff/evals/adapter.py @@ -11,23 +11,27 @@ import time from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from cliff.agents.runtime._prompts import build_user_prompt from cliff.agents.runtime.deps import WorkspaceDeps from cliff.agents.runtime.provider import build_model if TYPE_CHECKING: + from pydantic import BaseModel from pydantic_ai.models import Model from cliff.evals.registry import AgentEvalSpec +# A finding is a raw scanner dict (string keys); values are heterogeneous. +Finding = dict[str, object] + @dataclass class AgentRun: """A measured single run — the output plus what it cost (ADR-0050 §4).""" - output: Any + output: BaseModel input_tokens: int output_tokens: int total_tokens: int @@ -36,34 +40,52 @@ class AgentRun: async def _run( spec: AgentEvalSpec, - finding: dict[str, Any], + finding: Finding, *, env: dict[str, str] | None, model_id: str | None, model: Model | None, - prior_context: dict[str, dict[str, Any]] | None, + prior_context: dict[str, dict[str, object]] | None, ): - resolved_model = model if model is not None else build_model(env or {}, model_id) + # CI lane injects ``model``; live lane builds from env + the case/spec model + # (falling back to the spec's ``default_model`` when no id is supplied). + resolved_model = ( + model + if model is not None + else build_model(env or {}, model_id or spec.default_model) + ) agent = spec.build_agent(resolved_model) deps = WorkspaceDeps( workspace_id="eval", workspace_dir="/tmp/cliff-eval", - finding=finding, + finding=dict(finding), prior_context=prior_context or {}, env_vars=env or {}, ) return await agent.run(build_user_prompt(deps), deps=deps) +def _validated_output(spec: AgentEvalSpec, result) -> BaseModel: + """PA already validates against the agent's ``output_type``; assert it so a + misconfigured registry entry fails loudly rather than scoring garbage.""" + output = result.output + if not isinstance(output, spec.output_type): + raise TypeError( + f"{spec.name}: expected {spec.output_type.__name__}, got " + f"{type(output).__name__}" + ) + return output + + async def run_agent( spec: AgentEvalSpec, - finding: dict[str, Any], + finding: Finding, *, env: dict[str, str] | None = None, model_id: str | None = None, model: Model | None = None, - prior_context: dict[str, dict[str, Any]] | None = None, -) -> Any: + prior_context: dict[str, dict[str, object]] | None = None, +) -> BaseModel: """Run *spec*'s agent over a single eval case and return its output object. Provide ``model`` directly (CI lane: a ``FunctionModel``/``TestModel``) or @@ -74,17 +96,17 @@ async def run_agent( result = await _run( spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context ) - return result.output + return _validated_output(spec, result) async def run_agent_measured( spec: AgentEvalSpec, - finding: dict[str, Any], + finding: Finding, *, env: dict[str, str] | None = None, model_id: str | None = None, model: Model | None = None, - prior_context: dict[str, dict[str, Any]] | None = None, + prior_context: dict[str, dict[str, object]] | None = None, ) -> AgentRun: """Like :func:`run_agent`, but also returns token usage + wall-clock time so the runner can enforce a per-case / per-run budget.""" @@ -95,7 +117,7 @@ async def run_agent_measured( duration = time.monotonic() - start usage = result.usage return AgentRun( - output=result.output, + output=_validated_output(spec, result), input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0, total_tokens=usage.total_tokens or 0, diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index c8621273..8cfb06e9 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -17,6 +17,8 @@ from pydantic import BaseModel, Field +Tier = Literal["ci", "live"] + # Public synthetic sample lives in-repo; backend/ root is parents[2]. _SAMPLE_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval" @@ -25,9 +27,30 @@ def dataset_dir() -> Path: """Where datasets are read from (ADR-0050 hybrid: harness public, data private). Defaults to the public synthetic sample; the private eval project (``cliff-os/eval``) overrides it via ``CLIFF_EVAL_DATASET_DIR`` to point at - the real/confidential golden sets — which never enter this public repo.""" + the real/confidential golden sets — which never enter this public repo. + + A relative override is anchored to an absolute path (``.resolve()``) so the + same value resolves identically regardless of the process cwd.""" override = os.environ.get("CLIFF_EVAL_DATASET_DIR") - return Path(override) if override else _SAMPLE_DIR + return Path(override).expanduser().resolve() if override else _SAMPLE_DIR + + +class Expected(BaseModel): + """Golden labels for a case. A typed contract (not a free-form dict) so a + malformed JSONL row fails in ``load_cases`` instead of silently slipping a + bad shape past ``check_cve_ids`` / ``check_cvss_within``. Only declared + keys are graded — an omitted field means "no expectation".""" + + model_config = {"extra": "forbid"} + + cve_ids: list[str] | None = None + cvss_score: float | None = None + cvss_min: float | None = None + cvss_max: float | None = None + + def as_dict(self) -> dict[str, Any]: + """The declared-only mapping the deterministic evaluators consume.""" + return self.model_dump(exclude_unset=True) class EvalCase(BaseModel): @@ -36,14 +59,14 @@ class EvalCase(BaseModel): case where the agent MUST decline (no CVE / post-cutoff).""" id: str - tier: Literal["ci", "live"] = "live" + tier: Tier = "live" edge_case: str | None = None abstain: bool = False finding: dict[str, Any] - expected: dict[str, Any] = Field(default_factory=dict) + expected: Expected = Field(default_factory=Expected) -def load_cases(agent: str, *, tier: str | None = None) -> list[EvalCase]: +def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: """Load ``.jsonl`` from the active dataset dir into typed cases.""" path = dataset_dir() / f"{agent}.jsonl" if not path.is_file(): diff --git a/backend/cliff/evals/evaluators.py b/backend/cliff/evals/evaluators.py index d6b97bc8..14d17a89 100644 --- a/backend/cliff/evals/evaluators.py +++ b/backend/cliff/evals/evaluators.py @@ -28,7 +28,10 @@ re.compile(r"\bSNYK-[A-Z0-9-]+", re.IGNORECASE), re.compile(r"\bGHSA-[a-z0-9]{4}-", re.IGNORECASE), re.compile(r"^\s*\[[^\]]+\]"), # leading "[semgrep] ..." style tag - re.compile(r"\b[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_.-]+\b"), # dotted rule path + # Dotted scanner rule paths (e.g. javascript.lang.security.detect-eval). + # Require a letter in every segment so plain semantic versions like + # "2.17.1" are NOT flagged as jargon. + re.compile(r"\b[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*){2,}\b", re.IGNORECASE), ) @@ -61,11 +64,7 @@ async def assess_references(output: Any, *, http: Any = None) -> ReferenceAssess the network pass deterministically; the live lane leaves it ``None``. """ refs = getattr(output, "references", None) - check = ( - await clean_references(refs, http=http) - if http is not None - else await clean_references(refs) - ) + check = await clean_references(refs, http=http) # http=None → real HEADs structural = [(u, r) for u, r in check.dropped if not r.startswith("http_")] network = [(u, r) for u, r in check.dropped if r.startswith("http_")] return ReferenceAssessment(structural, network, check.kept) diff --git a/backend/cliff/evals/pricing.py b/backend/cliff/evals/pricing.py index f4e168ff..9581aac9 100644 --- a/backend/cliff/evals/pricing.py +++ b/backend/cliff/evals/pricing.py @@ -22,8 +22,11 @@ def _match(model_id: str | None) -> str | None: + """Longest matching key wins, so a more specific id (e.g. ``gpt-4o-mini``) + isn't mispriced against a shorter prefix (``gpt-4o``).""" m = (model_id or "").lower() - return next((k for k in _PRICE_USD_PER_MTOK if k in m), None) + candidates = [k for k in _PRICE_USD_PER_MTOK if k in m] + return max(candidates, key=len) if candidates else None def estimate_cost_usd( diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py index c5de1659..c26dc09f 100644 --- a/backend/cliff/evals/registry.py +++ b/backend/cliff/evals/registry.py @@ -8,28 +8,27 @@ first). Add entries as each agent's eval lands. """ -from __future__ import annotations +# No ``from __future__ import annotations`` here: Pydantic resolves these field +# annotations eagerly at class definition, so ``Callable`` / ``Model`` / ``Agent`` +# are genuinely runtime imports (and ruff TC-flags them only under future +# annotations). +from collections.abc import Callable -from dataclasses import dataclass -from typing import TYPE_CHECKING +from pydantic import BaseModel, ConfigDict +from pydantic_ai import Agent +from pydantic_ai.models import Model from cliff.agents.runtime.finding_enricher import build_agent as _build_enricher from cliff.agents.schemas import EnrichmentOutput -if TYPE_CHECKING: - from collections.abc import Callable - from pydantic import BaseModel - from pydantic_ai import Agent - from pydantic_ai.models import Model - - -@dataclass(frozen=True) -class BudgetSpec: +class BudgetSpec(BaseModel): """Budget ceilings, enforced by the runner (ADR-0050 §4). The token + time caps are reliable hard limits; ``*_usd`` is best-effort (skipped when the model isn't in the pricing table). A breach fails the eval run.""" + model_config = ConfigDict(frozen=True) + # Per case. max_usd: float | None = None max_tokens: int | None = None @@ -39,8 +38,11 @@ class BudgetSpec: max_run_tokens: int | None = None -@dataclass(frozen=True) -class AgentEvalSpec: +class AgentEvalSpec(BaseModel): + # ``build_agent`` (a callable) and ``output_type`` (a class) aren't Pydantic + # types, so allow arbitrary types; frozen for an immutable registry. + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + name: str build_agent: Callable[[Model], Agent] output_type: type[BaseModel] diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index bde8eec5..10443247 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -90,6 +90,12 @@ async def run_enricher_eval( Graded metrics (dataset-wide pass rate vs ``graded_floor``): cve_ids, cvss_within, no_jargon_title, reference_liveness. """ + if not cases: + raise ValueError( + "run_enricher_eval got 0 cases — an empty dataset must fail, not " + "silently report PASS. Check the dataset path / tier filter." + ) + spec = get_spec("finding_enricher") budget = spec.budget result = EvalRunResult( @@ -143,8 +149,9 @@ async def run_enricher_eval( if not ok: result.hard_failures.append(f"{case.id}: abstention — {reason}") - graded["cve_ids"].append(check_cve_ids(out, case.expected)[0]) - graded["cvss_within"].append(check_cvss_within(out, case.expected)[0]) + expected = case.expected.as_dict() + graded["cve_ids"].append(check_cve_ids(out, expected)[0]) + graded["cvss_within"].append(check_cvss_within(out, expected)[0]) graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) graded["reference_liveness"].append(not refs.network) diff --git a/backend/tests/agents/conftest.py b/backend/tests/agents/conftest.py index 85546c50..f28505de 100644 --- a/backend/tests/agents/conftest.py +++ b/backend/tests/agents/conftest.py @@ -12,12 +12,20 @@ import pytest -_api_key_set = bool( - os.environ.get("OPENAI_API_KEY") or os.environ.get("ANTHROPIC_API_KEY") +# Any provider build_model() can use — so a runnable non-OpenAI/Anthropic setup +# (e.g. OPENROUTER_API_KEY + CLIFF_EVAL_MODEL=openrouter/...) doesn't get skipped. +_PROVIDER_ENV_VARS = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "GEMINI_API_KEY", + "OLLAMA_BASE_URL", # Ollama authenticates with no key ) +_api_key_set = any(os.environ.get(v) for v in _PROVIDER_ENV_VARS) _skip_no_key = pytest.mark.skipif( - not _api_key_set, reason="No LLM API key set (OPENAI_API_KEY or ANTHROPIC_API_KEY)" + not _api_key_set, + reason=f"No LLM provider configured (one of: {', '.join(_PROVIDER_ENV_VARS)})", ) # Files whose every test hits a real LLM (the live evals). Everything else diff --git a/backend/tests/agents/test_evals_harness.py b/backend/tests/agents/test_evals_harness.py index 554c5ceb..693670ea 100644 --- a/backend/tests/agents/test_evals_harness.py +++ b/backend/tests/agents/test_evals_harness.py @@ -132,10 +132,10 @@ def test_dataset_loads_and_assertions_are_supported(): cases = load_cases("finding_enricher") assert len(cases) >= 5 assert {c.id for c in cases} >= {"log4shell", "semgrep-no-cve-eval"} - # every abstain case declares no expected cve_ids (the abstention contract) + # every abstain case declares an empty expected cve_ids (abstention contract) for c in cases: if c.abstain: - assert c.expected.get("cve_ids", []) == [] + assert c.expected.cve_ids == [] # registry advertises exactly the assertion families this agent's evaluators # implement (ADR-0050 §1: dataset assertions ⊆ supported_assertions) assert spec.supported_assertions == frozenset( From 543557fe524ff6d7b38ccaa1805a2651324197d3 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 18:04:19 +0300 Subject: [PATCH 6/7] fix(evals): address self-review findings (/code-review on #265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctness: - conftest: the widened live-LLM gate (prev commit) un-skipped live tests for Ollama/OpenRouter/Gemini-only hosts, but eval_model() only maps Anthropic else gpt-4o-mini -> those hosts hard-errored instead of skipping. FIX: gate + model selection now share one policy (cliff.evals.models.eval_runnable / select_eval_model) so they can't diverge; an Ollama-only host with no override skips cleanly; OpenRouter/Gemini auto-select a runnable model. - evaluators: the dotted-rule jargon regex flagged legit titles (domains like www.example.com, frameworks like asp.net.core.mvc). FIX: require a hyphen/ underscore in the run (real scanner rule ids have one) so only rule paths match. Altitude / cleanup: - Move model+env selection out of tests/eval_utils into the public cliff/evals/models.py so the private cliff-os/eval runner shares it instead of re-implementing (kills the cross-repo drift the helper exists to prevent). - registry: drop dead config (live_only, eval_frozen — nothing read them); soften the supported_assertions docstring (the runner doesn't yet validate per-case assertions — cases don't declare them). - adapter: rename the AgentRun measurement struct -> MeasuredRun (it shadowed the domain cliff.models.AgentRun used in ~20 files). - cases: load_cases hints at CLIFF_EVAL_DATASET_DIR when the (wheel-excluded) default sample dir is missing. - pricing: docstring is honest that an unpriced sub-model falls back to its nearest priced sibling (over-estimate; token cap is the hard limit). Verified: ruff clean, CI harness 9/9, 104 deterministic agent tests, live eval vs claude-haiku-4-5 PASS (no_jargon_title 100% with the regex fix). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/__init__.py | 4 ++ backend/cliff/evals/adapter.py | 8 ++-- backend/cliff/evals/cases.py | 11 +++++- backend/cliff/evals/evaluators.py | 11 ++++-- backend/cliff/evals/models.py | 59 ++++++++++++++++++++++++++++++ backend/cliff/evals/pricing.py | 7 +++- backend/cliff/evals/registry.py | 9 +++-- backend/tests/agents/conftest.py | 21 ++++------- backend/tests/agents/eval_utils.py | 34 +++++------------ 9 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 backend/cliff/evals/models.py diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py index d3907581..2e08e8d4 100644 --- a/backend/cliff/evals/__init__.py +++ b/backend/cliff/evals/__init__.py @@ -16,6 +16,7 @@ from cliff.evals.adapter import run_agent from cliff.evals.cases import EvalCase, dataset_dir, load_cases +from cliff.evals.models import eval_runnable, harvest_env, select_eval_model from cliff.evals.registry import AgentEvalSpec, get_spec from cliff.evals.runners import EvalRunResult, run_enricher_eval @@ -24,8 +25,11 @@ "EvalCase", "EvalRunResult", "dataset_dir", + "eval_runnable", "get_spec", + "harvest_env", "load_cases", "run_agent", "run_enricher_eval", + "select_eval_model", ] diff --git a/backend/cliff/evals/adapter.py b/backend/cliff/evals/adapter.py index ecbfe871..d2edbdeb 100644 --- a/backend/cliff/evals/adapter.py +++ b/backend/cliff/evals/adapter.py @@ -28,7 +28,7 @@ @dataclass -class AgentRun: +class MeasuredRun: """A measured single run — the output plus what it cost (ADR-0050 §4).""" output: BaseModel @@ -107,7 +107,7 @@ async def run_agent_measured( model_id: str | None = None, model: Model | None = None, prior_context: dict[str, dict[str, object]] | None = None, -) -> AgentRun: +) -> MeasuredRun: """Like :func:`run_agent`, but also returns token usage + wall-clock time so the runner can enforce a per-case / per-run budget.""" start = time.monotonic() @@ -116,7 +116,7 @@ async def run_agent_measured( ) duration = time.monotonic() - start usage = result.usage - return AgentRun( + return MeasuredRun( output=_validated_output(spec, result), input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0, @@ -125,4 +125,4 @@ async def run_agent_measured( ) -__all__ = ["AgentRun", "run_agent", "run_agent_measured"] +__all__ = ["MeasuredRun", "run_agent", "run_agent_measured"] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py index 8cfb06e9..f63e513d 100644 --- a/backend/cliff/evals/cases.py +++ b/backend/cliff/evals/cases.py @@ -70,7 +70,16 @@ def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: """Load ``.jsonl`` from the active dataset dir into typed cases.""" path = dataset_dir() / f"{agent}.jsonl" if not path.is_file(): - raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}") + hint = "" + if not os.environ.get("CLIFF_EVAL_DATASET_DIR"): + # The in-repo synthetic sample isn't packaged in the wheel (tests/ + # is excluded), so a wheel-installed consumer must point at its own + # dataset dir rather than rely on the default. + hint = ( + " — set CLIFF_EVAL_DATASET_DIR (the sample dataset ships only" + " in a source checkout, not the installed package)" + ) + raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}{hint}") cases: list[EvalCase] = [] for line_no, raw in enumerate(path.read_text().splitlines(), start=1): line = raw.strip() diff --git a/backend/cliff/evals/evaluators.py b/backend/cliff/evals/evaluators.py index 14d17a89..4c1316e8 100644 --- a/backend/cliff/evals/evaluators.py +++ b/backend/cliff/evals/evaluators.py @@ -29,9 +29,14 @@ re.compile(r"\bGHSA-[a-z0-9]{4}-", re.IGNORECASE), re.compile(r"^\s*\[[^\]]+\]"), # leading "[semgrep] ..." style tag # Dotted scanner rule paths (e.g. javascript.lang.security.detect-eval). - # Require a letter in every segment so plain semantic versions like - # "2.17.1" are NOT flagged as jargon. - re.compile(r"\b[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*){2,}\b", re.IGNORECASE), + # The leading lookahead requires a hyphen/underscore somewhere in the run, + # so semantic versions ("2.17.1") and plain dotted words/domains + # ("www.example.com", "asp.net.core.mvc") are NOT flagged — real scanner + # rule ids almost always carry a hyphen/underscore in a segment. + re.compile( + r"\b(?=[a-z0-9._-]*[_-])[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*){2,}\b", + re.IGNORECASE, + ), ) diff --git a/backend/cliff/evals/models.py b/backend/cliff/evals/models.py new file mode 100644 index 00000000..ed96cdb6 --- /dev/null +++ b/backend/cliff/evals/models.py @@ -0,0 +1,59 @@ +"""Eval model + provider selection (ADR-0050). + +Lives in the public ``cliff.evals`` package (not ``tests/``) so the in-repo +live test AND the private ``cliff-os/eval`` runner share ONE selection policy — +otherwise the two lanes can silently run different models and mask each other's +regressions. + +``select_eval_model`` returns the cheap-but-capable model for whichever provider +is configured, or ``None`` when nothing runnable is available. The live-test skip +gate is exactly "is there a runnable model?" — so it can't diverge from what the +runner will actually use (the conftest-gate-vs-eval_model bug this replaces). +""" + +from __future__ import annotations + +import os + +# Cheap-capable default per provider, keyed by the env var that selects it. +# Order matters: prefer the production default (Anthropic haiku) first. +_PROVIDER_DEFAULTS: tuple[tuple[str, str], ...] = ( + ("ANTHROPIC_API_KEY", "anthropic/claude-haiku-4-5"), + ("OPENAI_API_KEY", "openai/gpt-4o-mini"), + ("OPENROUTER_API_KEY", "openrouter/anthropic/claude-haiku-4.5"), + ("GEMINI_API_KEY", "google/gemini-2.5-flash"), + # Ollama needs a locally-pulled model name we can't assume, so it is NOT + # auto-selected — set CLIFF_EVAL_MODEL=ollama/ explicitly. +) + + +def harvest_env(source: dict[str, str] | None = None) -> dict[str, str]: + """The provider credentials the model factory reads (``*_API_KEY`` / + ``*_BASE_URL``), harvested from the host env.""" + env = source if source is not None else os.environ + return {k: v for k, v in env.items() if k.endswith(("_API_KEY", "_BASE_URL"))} + + +def select_eval_model(source: dict[str, str] | None = None) -> str | None: + """The model the eval should run, or ``None`` if nothing is runnable. + + ``CLIFF_EVAL_MODEL`` overrides (the operator owns matching the key); else the + first configured provider's cheap default. Match the production default + (Anthropic haiku) so the eval predicts customer experience. + """ + env = source if source is not None else os.environ + if override := env.get("CLIFF_EVAL_MODEL"): + return override + for key_var, model in _PROVIDER_DEFAULTS: + if env.get(key_var): + return model + return None + + +def eval_runnable(source: dict[str, str] | None = None) -> bool: + """True iff a live eval can actually run (a model resolves). The live-test + skip gate — kept identical to ``select_eval_model`` so they never diverge.""" + return select_eval_model(source) is not None + + +__all__ = ["eval_runnable", "harvest_env", "select_eval_model"] diff --git a/backend/cliff/evals/pricing.py b/backend/cliff/evals/pricing.py index 9581aac9..7a30a472 100644 --- a/backend/cliff/evals/pricing.py +++ b/backend/cliff/evals/pricing.py @@ -22,8 +22,11 @@ def _match(model_id: str | None) -> str | None: - """Longest matching key wins, so a more specific id (e.g. ``gpt-4o-mini``) - isn't mispriced against a shorter prefix (``gpt-4o``).""" + """Longest matching key wins, so a model with its OWN key isn't mispriced + against a shorter prefix. A sub-model NOT in the table still falls back to + its nearest priced sibling (e.g. ``gpt-5-mini`` → ``gpt-5``) — an + over-estimate, which is the safe direction for a best-effort budget guard. + Add the sub-model's own key when an exact price matters.""" m = (model_id or "").lower() candidates = [k for k in _PRICE_USD_PER_MTOK if k in m] return max(candidates, key=len) if candidates else None diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py index c26dc09f..75c72304 100644 --- a/backend/cliff/evals/registry.py +++ b/backend/cliff/evals/registry.py @@ -1,8 +1,11 @@ """Per-agent eval registry (ADR-0050 §1). One ``AgentEvalSpec`` per eval target — the single source of truth for what -each agent supports, its budget, and how to build it. The runner validates a -dataset's declared assertions against ``supported_assertions``. +each agent supports, its budget, and how to build it. ``supported_assertions`` +documents which assertion families are meaningful for the agent; per-case +datasets don't yet declare assertions (the per-agent runner applies a fixed +set), so it is descriptive, not yet runner-enforced — that validation lands +when cases carry explicit per-case assertions. Only ``finding_enricher`` is wired today (ADR-0050 rollout §7: highest-risk first). Add entries as each agent's eval lands. @@ -50,8 +53,6 @@ class AgentEvalSpec(BaseModel): supported_assertions: frozenset[str] budget: BudgetSpec default_model: str | None = None - live_only: bool = False - eval_frozen: bool = False # deprecated agents (owner_resolver): keep, don't maintain _REGISTRY: dict[str, AgentEvalSpec] = { diff --git a/backend/tests/agents/conftest.py b/backend/tests/agents/conftest.py index f28505de..8eb4bfa5 100644 --- a/backend/tests/agents/conftest.py +++ b/backend/tests/agents/conftest.py @@ -8,24 +8,17 @@ from __future__ import annotations -import os - import pytest -# Any provider build_model() can use — so a runnable non-OpenAI/Anthropic setup -# (e.g. OPENROUTER_API_KEY + CLIFF_EVAL_MODEL=openrouter/...) doesn't get skipped. -_PROVIDER_ENV_VARS = ( - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "OPENROUTER_API_KEY", - "GEMINI_API_KEY", - "OLLAMA_BASE_URL", # Ollama authenticates with no key -) -_api_key_set = any(os.environ.get(v) for v in _PROVIDER_ENV_VARS) +from cliff.evals.models import eval_runnable +# Gate live-LLM tests on whether a model actually resolves (CLIFF_EVAL_MODEL +# override or a configured provider) — derived from the SAME policy the runner +# uses, so the gate can't say "runnable" while eval_model picks a model with no +# key (e.g. an Ollama-only host that would otherwise hard-error, not skip). _skip_no_key = pytest.mark.skipif( - not _api_key_set, - reason=f"No LLM provider configured (one of: {', '.join(_PROVIDER_ENV_VARS)})", + not eval_runnable(), + reason="No runnable eval model (set a provider key or CLIFF_EVAL_MODEL)", ) # Files whose every test hits a real LLM (the live evals). Everything else diff --git a/backend/tests/agents/eval_utils.py b/backend/tests/agents/eval_utils.py index b96d3e2e..b2797763 100644 --- a/backend/tests/agents/eval_utils.py +++ b/backend/tests/agents/eval_utils.py @@ -1,30 +1,16 @@ -"""Shared setup for the real-LLM agent evals (IMPL-0022 PR #3b). +"""Shared setup for the real-LLM agent evals. -The two live-LLM test modules — ``test_normalizer_agent`` and -``test_plain_description_eval`` — both need the same provider env + a -capable-but-cheap model id. Keeping that selection policy here means it -can't drift between the two files. Both modules are skip-gated on an API -key being present (see ``conftest.py``). +Thin re-export of the canonical selection policy in ``cliff.evals.models`` — +which lives in the public package (not ``tests/``) so the in-repo live tests AND +the private ``cliff-os/eval`` runner share ONE policy and can't drift. Live test +modules are skip-gated on a runnable model being available (see ``conftest.py``). """ from __future__ import annotations -import os +from cliff.evals.models import harvest_env, select_eval_model -# Provider credentials for the app-level normalizer, harvested from the host -# env (the same shape the running daemon resolves into a workspace). -LLM_ENV = { - k: v for k, v in os.environ.items() if k.endswith(("_API_KEY", "_BASE_URL")) -} - - -def eval_model() -> str: - """Capable, cheap model for the real-LLM eval (override: ``CLIFF_EVAL_MODEL``).""" - if override := os.environ.get("CLIFF_EVAL_MODEL"): - return override - if os.environ.get("ANTHROPIC_API_KEY"): - return "anthropic/claude-haiku-4-5" - return "openai/gpt-4o-mini" - - -LLM_MODEL = eval_model() +LLM_ENV = harvest_env() +# Resolved at import; only *used* by tests that are skip-gated on a runnable +# model, so the last-resort fallback is just to keep this non-None at import. +LLM_MODEL = select_eval_model() or "openai/gpt-4o-mini" From 5cf5f9a1f5fcf38aad327b09a6d06f86cfe7cba2 Mon Sep 17 00:00:00 2001 From: Gal Ankonina Date: Tue, 9 Jun 2026 18:10:06 +0300 Subject: [PATCH 7/7] fix(evals): price USD budget against the resolved model (CodeRabbit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit estimate_cost_usd was called with the raw model_id, which can be None when the caller relies on the adapter's default — silently skipping the USD caps. Price against `model_id or spec.default_model` (what the adapter actually runs), and set finding_enricher's default_model to the production default (anthropic/claude-haiku-4-5) so the fallback + pricing are both real. The other open threads were already addressed in prior commits (redundant http branch removed, conftest gate now shares eval_runnable across all providers, empty-cases now raises) and one is a standing decline (run_agent vs run_no_tools_agent — different return shapes, ~2 shared lines). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/cliff/evals/registry.py | 3 +++ backend/cliff/evals/runners.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py index 75c72304..d6adcd42 100644 --- a/backend/cliff/evals/registry.py +++ b/backend/cliff/evals/registry.py @@ -77,6 +77,9 @@ class AgentEvalSpec(BaseModel): max_run_usd=0.50, max_run_tokens=120_000, ), + # Match the production default so the eval predicts customer experience; + # used by the adapter (and USD pricing) when a caller omits model_id. + default_model="anthropic/claude-haiku-4-5", ), } diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py index 10443247..4e17233f 100644 --- a/backend/cliff/evals/runners.py +++ b/backend/cliff/evals/runners.py @@ -112,6 +112,10 @@ async def run_enricher_eval( "no_jargon_title": [], "reference_liveness": [], } + # Price against the model the adapter actually runs (it falls back to + # spec.default_model when model_id is None), so a None id can't silently + # skip the USD caps. + pricing_model_id = model_id or spec.default_model run_cost = 0.0 run_cost_known = True @@ -129,7 +133,9 @@ async def run_enricher_eval( result.budget_failures.append( f"{case.id}: {run.duration_s:.0f}s > {budget.max_duration_s:.0f}s cap" ) - case_cost = estimate_cost_usd(model_id, run.input_tokens, run.output_tokens) + case_cost = estimate_cost_usd( + pricing_model_id, run.input_tokens, run.output_tokens + ) if case_cost is None: run_cost_known = False else: