feat(evals): agent eval harness + finding_enricher reference implementation (ADR-0050)

backend/cliff/evals/__init__.py

@@ -0,0 +1,35 @@
+"""Cliff agent-evaluation harness (ADR-0050).
+
+A thin, generic layer over ``pydantic-evals``: a per-agent registry, an
+adapter that drives any agent through one call, and a small set of custom
+evaluators. Datasets live as JSONL under ``backend/tests/agents/eval/``.
+
+Two lanes (ADR-0050 §5):
+
+* **CI** — deterministic, ``FunctionModel``/``TestModel``, every push. Proves
+  the evaluators + adapter are correct without a key.
+* **Live** — key-gated, real model, measures actual agent quality.
+
+The first agent wired is ``finding_enricher`` (the reference implementation,
+ADR-0050 rollout §7).
+"""
+
+from cliff.evals.adapter import run_agent
+from cliff.evals.cases import EvalCase, dataset_dir, load_cases
+from cliff.evals.models import eval_runnable, harvest_env, select_eval_model
+from cliff.evals.registry import AgentEvalSpec, get_spec
+from cliff.evals.runners import EvalRunResult, run_enricher_eval
+
+__all__ = [
+    "AgentEvalSpec",
+    "EvalCase",
+    "EvalRunResult",
+    "dataset_dir",
+    "eval_runnable",
+    "get_spec",
+    "harvest_env",
+    "load_cases",
+    "run_agent",
+    "run_enricher_eval",
+    "select_eval_model",
+]

backend/cliff/evals/adapter.py

@@ -0,0 +1,128 @@
+"""Generic agent adapter for the eval harness (ADR-0050 §1).
+
+One call drives any workspace-scoped runtime agent: build the model, build
+the agent, construct ``WorkspaceDeps`` from the case input, render the same
+user prompt the executor uses, and run. The model can be injected (a
+``FunctionModel`` for the deterministic CI lane) or built from canonical AI
+state (the live lane).
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from cliff.agents.runtime._prompts import build_user_prompt
+from cliff.agents.runtime.deps import WorkspaceDeps
+from cliff.agents.runtime.provider import build_model
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+    from pydantic_ai.models import Model
+
+    from cliff.evals.registry import AgentEvalSpec
+
+# A finding is a raw scanner dict (string keys); values are heterogeneous.
+Finding = dict[str, object]
+
+
+@dataclass
+class MeasuredRun:
+    """A measured single run — the output plus what it cost (ADR-0050 §4)."""
+
+    output: BaseModel
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+    duration_s: float
+
+
+async def _run(
+    spec: AgentEvalSpec,
+    finding: Finding,
+    *,
+    env: dict[str, str] | None,
+    model_id: str | None,
+    model: Model | None,
+    prior_context: dict[str, dict[str, object]] | None,
+):
+    # CI lane injects ``model``; live lane builds from env + the case/spec model
+    # (falling back to the spec's ``default_model`` when no id is supplied).
+    resolved_model = (
+        model
+        if model is not None
+        else build_model(env or {}, model_id or spec.default_model)
+    )
+    agent = spec.build_agent(resolved_model)
+    deps = WorkspaceDeps(
+        workspace_id="eval",
+        workspace_dir="/tmp/cliff-eval",
+        finding=dict(finding),
+        prior_context=prior_context or {},
+        env_vars=env or {},
+    )
+    return await agent.run(build_user_prompt(deps), deps=deps)
+
+
+def _validated_output(spec: AgentEvalSpec, result) -> BaseModel:
+    """PA already validates against the agent's ``output_type``; assert it so a
+    misconfigured registry entry fails loudly rather than scoring garbage."""
+    output = result.output
+    if not isinstance(output, spec.output_type):
+        raise TypeError(
+            f"{spec.name}: expected {spec.output_type.__name__}, got "
+            f"{type(output).__name__}"
+        )
+    return output
+
+
+async def run_agent(
+    spec: AgentEvalSpec,
+    finding: Finding,
+    *,
+    env: dict[str, str] | None = None,
+    model_id: str | None = None,
+    model: Model | None = None,
+    prior_context: dict[str, dict[str, object]] | None = None,
+) -> BaseModel:
+    """Run *spec*'s agent over a single eval case and return its output object.
+
+    Provide ``model`` directly (CI lane: a ``FunctionModel``/``TestModel``) or
+    ``env`` + ``model_id`` to build a real model (live lane). The returned
+    object is the agent's structured ``output_type`` instance (e.g.
+    ``EnrichmentOutput``), exactly what evaluators score.
+    """
+    result = await _run(
+        spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context
+    )
+    return _validated_output(spec, result)
+
+
+async def run_agent_measured(
+    spec: AgentEvalSpec,
+    finding: Finding,
+    *,
+    env: dict[str, str] | None = None,
+    model_id: str | None = None,
+    model: Model | None = None,
+    prior_context: dict[str, dict[str, object]] | None = None,
+) -> MeasuredRun:
+    """Like :func:`run_agent`, but also returns token usage + wall-clock time so
+    the runner can enforce a per-case / per-run budget."""
+    start = time.monotonic()
+    result = await _run(
+        spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context
+    )
+    duration = time.monotonic() - start
+    usage = result.usage
+    return MeasuredRun(
+        output=_validated_output(spec, result),
+        input_tokens=usage.input_tokens or 0,
+        output_tokens=usage.output_tokens or 0,
+        total_tokens=usage.total_tokens or 0,
+        duration_s=duration,
+    )
+
+
+__all__ = ["MeasuredRun", "run_agent", "run_agent_measured"]

backend/cliff/evals/cases.py

@@ -0,0 +1,97 @@
+"""Eval dataset cases (ADR-0050 §2).
+
+One typed schema for every agent's cases, stored as JSONL at
+``backend/tests/agents/eval/<agent>.jsonl`` — one case per line, append a
+line to add a case. ``load_cases`` enumerates them in file order.
+
+(The dataset lives under ``tests/`` and is only read by the eval tests; the
+loader resolving a ``tests/`` path from a ``cliff.*`` module is the
+test/prod-line blur tracked as ADR-0050 Open question #7.)
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+Tier = Literal["ci", "live"]
+
+# Public synthetic sample lives in-repo; backend/ root is parents[2].
+_SAMPLE_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval"
+
+
+def dataset_dir() -> Path:
+    """Where datasets are read from (ADR-0050 hybrid: harness public, data
+    private). Defaults to the public synthetic sample; the private eval project
+    (``cliff-os/eval``) overrides it via ``CLIFF_EVAL_DATASET_DIR`` to point at
+    the real/confidential golden sets — which never enter this public repo.
+
+    A relative override is anchored to an absolute path (``.resolve()``) so the
+    same value resolves identically regardless of the process cwd."""
+    override = os.environ.get("CLIFF_EVAL_DATASET_DIR")
+    return Path(override).expanduser().resolve() if override else _SAMPLE_DIR
+
+
+class Expected(BaseModel):
+    """Golden labels for a case. A typed contract (not a free-form dict) so a
+    malformed JSONL row fails in ``load_cases`` instead of silently slipping a
+    bad shape past ``check_cve_ids`` / ``check_cvss_within``. Only declared
+    keys are graded — an omitted field means "no expectation"."""
+
+    model_config = {"extra": "forbid"}
+
+    cve_ids: list[str] | None = None
+    cvss_score: float | None = None
+    cvss_min: float | None = None
+    cvss_max: float | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        """The declared-only mapping the deterministic evaluators consume."""
+        return self.model_dump(exclude_unset=True)
+
+
+class EvalCase(BaseModel):
+    """One eval case. ``finding`` is the raw input; ``expected`` holds the
+    golden labels the deterministic evaluators check; ``abstain`` marks a
+    case where the agent MUST decline (no CVE / post-cutoff)."""
+
+    id: str
+    tier: Tier = "live"
+    edge_case: str | None = None
+    abstain: bool = False
+    finding: dict[str, Any]
+    expected: Expected = Field(default_factory=Expected)
+
+
+def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]:
+    """Load ``<agent>.jsonl`` from the active dataset dir into typed cases."""
+    path = dataset_dir() / f"{agent}.jsonl"
+    if not path.is_file():
+        hint = ""
+        if not os.environ.get("CLIFF_EVAL_DATASET_DIR"):
+            # The in-repo synthetic sample isn't packaged in the wheel (tests/
+            # is excluded), so a wheel-installed consumer must point at its own
+            # dataset dir rather than rely on the default.
+            hint = (
+                " — set CLIFF_EVAL_DATASET_DIR (the sample dataset ships only"
+                " in a source checkout, not the installed package)"
+            )
+        raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}{hint}")
+    cases: list[EvalCase] = []
+    for line_no, raw in enumerate(path.read_text().splitlines(), start=1):
+        line = raw.strip()
+        if not line or line.startswith("//"):
+            continue
+        try:
+            cases.append(EvalCase.model_validate_json(line))
+        except ValueError as exc:  # malformed line — surface which one
+            raise ValueError(f"{path.name}:{line_no}: invalid case — {exc}") from exc
+    if tier is not None:
+        cases = [c for c in cases if c.tier == tier]
+    return cases
+
+
+__all__ = ["EvalCase", "load_cases"]