diff --git a/backend/cliff/evals/__init__.py b/backend/cliff/evals/__init__.py new file mode 100644 index 00000000..2e08e8d4 --- /dev/null +++ b/backend/cliff/evals/__init__.py @@ -0,0 +1,35 @@ +"""Cliff agent-evaluation harness (ADR-0050). + +A thin, generic layer over ``pydantic-evals``: a per-agent registry, an +adapter that drives any agent through one call, and a small set of custom +evaluators. Datasets live as JSONL under ``backend/tests/agents/eval/``. + +Two lanes (ADR-0050 §5): + +* **CI** — deterministic, ``FunctionModel``/``TestModel``, every push. Proves + the evaluators + adapter are correct without a key. +* **Live** — key-gated, real model, measures actual agent quality. + +The first agent wired is ``finding_enricher`` (the reference implementation, +ADR-0050 rollout §7). +""" + +from cliff.evals.adapter import run_agent +from cliff.evals.cases import EvalCase, dataset_dir, load_cases +from cliff.evals.models import eval_runnable, harvest_env, select_eval_model +from cliff.evals.registry import AgentEvalSpec, get_spec +from cliff.evals.runners import EvalRunResult, run_enricher_eval + +__all__ = [ + "AgentEvalSpec", + "EvalCase", + "EvalRunResult", + "dataset_dir", + "eval_runnable", + "get_spec", + "harvest_env", + "load_cases", + "run_agent", + "run_enricher_eval", + "select_eval_model", +] diff --git a/backend/cliff/evals/adapter.py b/backend/cliff/evals/adapter.py new file mode 100644 index 00000000..d2edbdeb --- /dev/null +++ b/backend/cliff/evals/adapter.py @@ -0,0 +1,128 @@ +"""Generic agent adapter for the eval harness (ADR-0050 §1). + +One call drives any workspace-scoped runtime agent: build the model, build +the agent, construct ``WorkspaceDeps`` from the case input, render the same +user prompt the executor uses, and run. The model can be injected (a +``FunctionModel`` for the deterministic CI lane) or built from canonical AI +state (the live lane). +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from cliff.agents.runtime._prompts import build_user_prompt +from cliff.agents.runtime.deps import WorkspaceDeps +from cliff.agents.runtime.provider import build_model + +if TYPE_CHECKING: + from pydantic import BaseModel + from pydantic_ai.models import Model + + from cliff.evals.registry import AgentEvalSpec + +# A finding is a raw scanner dict (string keys); values are heterogeneous. +Finding = dict[str, object] + + +@dataclass +class MeasuredRun: + """A measured single run — the output plus what it cost (ADR-0050 §4).""" + + output: BaseModel + input_tokens: int + output_tokens: int + total_tokens: int + duration_s: float + + +async def _run( + spec: AgentEvalSpec, + finding: Finding, + *, + env: dict[str, str] | None, + model_id: str | None, + model: Model | None, + prior_context: dict[str, dict[str, object]] | None, +): + # CI lane injects ``model``; live lane builds from env + the case/spec model + # (falling back to the spec's ``default_model`` when no id is supplied). + resolved_model = ( + model + if model is not None + else build_model(env or {}, model_id or spec.default_model) + ) + agent = spec.build_agent(resolved_model) + deps = WorkspaceDeps( + workspace_id="eval", + workspace_dir="/tmp/cliff-eval", + finding=dict(finding), + prior_context=prior_context or {}, + env_vars=env or {}, + ) + return await agent.run(build_user_prompt(deps), deps=deps) + + +def _validated_output(spec: AgentEvalSpec, result) -> BaseModel: + """PA already validates against the agent's ``output_type``; assert it so a + misconfigured registry entry fails loudly rather than scoring garbage.""" + output = result.output + if not isinstance(output, spec.output_type): + raise TypeError( + f"{spec.name}: expected {spec.output_type.__name__}, got " + f"{type(output).__name__}" + ) + return output + + +async def run_agent( + spec: AgentEvalSpec, + finding: Finding, + *, + env: dict[str, str] | None = None, + model_id: str | None = None, + model: Model | None = None, + prior_context: dict[str, dict[str, object]] | None = None, +) -> BaseModel: + """Run *spec*'s agent over a single eval case and return its output object. + + Provide ``model`` directly (CI lane: a ``FunctionModel``/``TestModel``) or + ``env`` + ``model_id`` to build a real model (live lane). The returned + object is the agent's structured ``output_type`` instance (e.g. + ``EnrichmentOutput``), exactly what evaluators score. + """ + result = await _run( + spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context + ) + return _validated_output(spec, result) + + +async def run_agent_measured( + spec: AgentEvalSpec, + finding: Finding, + *, + env: dict[str, str] | None = None, + model_id: str | None = None, + model: Model | None = None, + prior_context: dict[str, dict[str, object]] | None = None, +) -> MeasuredRun: + """Like :func:`run_agent`, but also returns token usage + wall-clock time so + the runner can enforce a per-case / per-run budget.""" + start = time.monotonic() + result = await _run( + spec, finding, env=env, model_id=model_id, model=model, prior_context=prior_context + ) + duration = time.monotonic() - start + usage = result.usage + return MeasuredRun( + output=_validated_output(spec, result), + input_tokens=usage.input_tokens or 0, + output_tokens=usage.output_tokens or 0, + total_tokens=usage.total_tokens or 0, + duration_s=duration, + ) + + +__all__ = ["MeasuredRun", "run_agent", "run_agent_measured"] diff --git a/backend/cliff/evals/cases.py b/backend/cliff/evals/cases.py new file mode 100644 index 00000000..f63e513d --- /dev/null +++ b/backend/cliff/evals/cases.py @@ -0,0 +1,97 @@ +"""Eval dataset cases (ADR-0050 §2). + +One typed schema for every agent's cases, stored as JSONL at +``backend/tests/agents/eval/.jsonl`` — one case per line, append a +line to add a case. ``load_cases`` enumerates them in file order. + +(The dataset lives under ``tests/`` and is only read by the eval tests; the +loader resolving a ``tests/`` path from a ``cliff.*`` module is the +test/prod-line blur tracked as ADR-0050 Open question #7.) +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, Field + +Tier = Literal["ci", "live"] + +# Public synthetic sample lives in-repo; backend/ root is parents[2]. +_SAMPLE_DIR = Path(__file__).resolve().parents[2] / "tests" / "agents" / "eval" + + +def dataset_dir() -> Path: + """Where datasets are read from (ADR-0050 hybrid: harness public, data + private). Defaults to the public synthetic sample; the private eval project + (``cliff-os/eval``) overrides it via ``CLIFF_EVAL_DATASET_DIR`` to point at + the real/confidential golden sets — which never enter this public repo. + + A relative override is anchored to an absolute path (``.resolve()``) so the + same value resolves identically regardless of the process cwd.""" + override = os.environ.get("CLIFF_EVAL_DATASET_DIR") + return Path(override).expanduser().resolve() if override else _SAMPLE_DIR + + +class Expected(BaseModel): + """Golden labels for a case. A typed contract (not a free-form dict) so a + malformed JSONL row fails in ``load_cases`` instead of silently slipping a + bad shape past ``check_cve_ids`` / ``check_cvss_within``. Only declared + keys are graded — an omitted field means "no expectation".""" + + model_config = {"extra": "forbid"} + + cve_ids: list[str] | None = None + cvss_score: float | None = None + cvss_min: float | None = None + cvss_max: float | None = None + + def as_dict(self) -> dict[str, Any]: + """The declared-only mapping the deterministic evaluators consume.""" + return self.model_dump(exclude_unset=True) + + +class EvalCase(BaseModel): + """One eval case. ``finding`` is the raw input; ``expected`` holds the + golden labels the deterministic evaluators check; ``abstain`` marks a + case where the agent MUST decline (no CVE / post-cutoff).""" + + id: str + tier: Tier = "live" + edge_case: str | None = None + abstain: bool = False + finding: dict[str, Any] + expected: Expected = Field(default_factory=Expected) + + +def load_cases(agent: str, *, tier: Tier | None = None) -> list[EvalCase]: + """Load ``.jsonl`` from the active dataset dir into typed cases.""" + path = dataset_dir() / f"{agent}.jsonl" + if not path.is_file(): + hint = "" + if not os.environ.get("CLIFF_EVAL_DATASET_DIR"): + # The in-repo synthetic sample isn't packaged in the wheel (tests/ + # is excluded), so a wheel-installed consumer must point at its own + # dataset dir rather than rely on the default. + hint = ( + " — set CLIFF_EVAL_DATASET_DIR (the sample dataset ships only" + " in a source checkout, not the installed package)" + ) + raise FileNotFoundError(f"No eval dataset for {agent!r} at {path}{hint}") + cases: list[EvalCase] = [] + for line_no, raw in enumerate(path.read_text().splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("//"): + continue + try: + cases.append(EvalCase.model_validate_json(line)) + except ValueError as exc: # malformed line — surface which one + raise ValueError(f"{path.name}:{line_no}: invalid case — {exc}") from exc + if tier is not None: + cases = [c for c in cases if c.tier == tier] + return cases + + +__all__ = ["EvalCase", "load_cases"] diff --git a/backend/cliff/evals/evaluators.py b/backend/cliff/evals/evaluators.py new file mode 100644 index 00000000..4c1316e8 --- /dev/null +++ b/backend/cliff/evals/evaluators.py @@ -0,0 +1,218 @@ +"""Evaluators for the finding_enricher (ADR-0050 §3). + +"Code first, judge last." Everything here is deterministic. The headline gate +is ``citation_liveness``: it reuses the production reference verifier +(``services.reference_verifier.clean_references``) — if that verifier would +*drop* any reference the agent emitted (structurally bogus GHSA/SHA, or a URL +the host 404s), the agent fabricated a citation and the case FAILS. Production +already strips these before the user sees them; the eval measures how often the +model tries. + +Each check is a pure ``(passed: bool, reason: str)`` function (clear pytest +messages, trivially unit-tested). The ``pydantic-evals`` ``Evaluator`` wrappers +at the bottom call them so the Dataset/Report path (ADR-0050 §1) works too. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from cliff.services.reference_verifier import clean_references + +# Scanner-specific jargon that a normalized title must not carry (ADR-0040: +# "a human should understand the title without knowing which scanner produced +# it"). Raw rule ids, leading bracket tags, dotted semgrep rule paths. +_JARGON_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bSNYK-[A-Z0-9-]+", re.IGNORECASE), + re.compile(r"\bGHSA-[a-z0-9]{4}-", re.IGNORECASE), + re.compile(r"^\s*\[[^\]]+\]"), # leading "[semgrep] ..." style tag + # Dotted scanner rule paths (e.g. javascript.lang.security.detect-eval). + # The leading lookahead requires a hyphen/underscore somewhere in the run, + # so semantic versions ("2.17.1") and plain dotted words/domains + # ("www.example.com", "asp.net.core.mvc") are NOT flagged — real scanner + # rule ids almost always carry a hyphen/underscore in a segment. + re.compile( + r"\b(?=[a-z0-9._-]*[_-])[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*){2,}\b", + re.IGNORECASE, + ), +) + + +def _norm_cves(values: Any) -> set[str]: + if not isinstance(values, (list, tuple)): + return set() + return {str(v).strip().upper() for v in values if str(v).strip()} + + +@dataclass +class ReferenceAssessment: + """Classification of the verifier's dropped references. + + ``structural`` = structurally-impossible (fabricated GHSA id, garbled + commit SHA, non-http, unsafe IP) — pure fabrication, caught in the + no-network sanitize pass. ``network`` = a plausible URL the host 404/410s + — a dead-link guess. They are scored differently (ADR-0050 §3): structural + is the zero-tolerance hard gate; network is a graded liveness metric. + """ + + structural: list[tuple[str, str]] + network: list[tuple[str, str]] + kept: list[str] + + +async def assess_references(output: Any, *, http: Any = None) -> ReferenceAssessment: + """Run the production verifier once and classify its drops. + + ``http`` (an ``httpx.AsyncClient``) is injectable so the CI lane can drive + the network pass deterministically; the live lane leaves it ``None``. + """ + refs = getattr(output, "references", None) + check = await clean_references(refs, http=http) # http=None → real HEADs + structural = [(u, r) for u, r in check.dropped if not r.startswith("http_")] + network = [(u, r) for u, r in check.dropped if r.startswith("http_")] + return ReferenceAssessment(structural, network, check.kept) + + +async def check_structural_citations(output: Any, *, http: Any = None) -> tuple[bool, str]: + """HARD GATE: FAIL on a structurally-impossible (fabricated) reference.""" + a = await assess_references(output, http=http) + if a.structural: + bad = "; ".join(f"{u} ({why})" for u, why in a.structural) + return False, f"fabricated references: {bad}" + return True, f"{len(a.kept)} ok, {len(a.network)} dead-link(s)" + + +async def check_reference_liveness(output: Any, *, http: Any = None) -> tuple[bool, str]: + """GRADED: FAIL if any reference 404/410s (a plausible-but-dead URL guess — + production strips these, so this measures the guess rate, not user impact).""" + a = await assess_references(output, http=http) + if a.network: + dead = "; ".join(u for u, _ in a.network) + return False, f"dead links: {dead}" + return True, f"all {len(a.kept)} reference(s) live" + + +def check_cve_ids(output: Any, expected: dict[str, Any]) -> tuple[bool, str]: + """Exact set match against the golden CVE ids (case-insensitive). No + ``cve_ids`` key declared → no expectation → pass (the abstention gate, not + this one, enforces emptiness on no-CVE cases).""" + if "cve_ids" not in expected: + return True, "no cve_ids expectation" + got = _norm_cves(getattr(output, "cve_ids", None)) + want = _norm_cves(expected.get("cve_ids", [])) + if got == want: + return True, f"cve_ids match ({sorted(want) or 'none'})" + return False, f"cve_ids mismatch: got {sorted(got)}, expected {sorted(want)}" + + +def check_cvss_within( + output: Any, expected: dict[str, Any], *, tol: float = 0.5 +) -> tuple[bool, str]: + """CVSS within ±tol of golden, or inside [cvss_min, cvss_max], or null + when the case expects abstention. No expectation declared → pass.""" + got = getattr(output, "cvss_score", None) + if "cvss_score" in expected: + want = expected["cvss_score"] + if want is None: + return (got is None), f"cvss expected null, got {got}" + if got is None: + return False, f"cvss expected ~{want}, got null" + ok = abs(float(got) - float(want)) <= tol + return ok, f"cvss {got} vs {want} (±{tol})" + if "cvss_min" in expected or "cvss_max" in expected: + lo = expected.get("cvss_min", 0.0) + hi = expected.get("cvss_max", 10.0) + if got is None: + return False, f"cvss expected in [{lo},{hi}], got null" + return (lo <= float(got) <= hi), f"cvss {got} in [{lo},{hi}]" + return True, "no cvss expectation" + + +def check_no_jargon_title(output: Any) -> tuple[bool, str]: + """normalized_title must carry no scanner jargon.""" + title = getattr(output, "normalized_title", None) + if not isinstance(title, str) or not title.strip(): + return False, "normalized_title missing/empty" + for pat in _JARGON_PATTERNS: + m = pat.search(title) + if m: + return False, f"title contains scanner jargon {m.group(0)!r}: {title!r}" + return True, "title clean" + + +def check_abstention(output: Any) -> tuple[bool, str]: + """For a no-CVE / post-cutoff case: no invented CVE id and no invented + CVSS score (the agent must decline rather than fabricate).""" + cves = _norm_cves(getattr(output, "cve_ids", None)) + cvss = getattr(output, "cvss_score", None) + if cves: + return False, f"should abstain but invented cve_ids {sorted(cves)}" + if cvss is not None: + return False, f"should abstain but invented cvss_score {cvss}" + return True, "abstained (no invented CVE/CVSS)" + + +# --- pydantic-evals Evaluator wrappers (Dataset/Report path) ----------------- + +try: # pydantic-evals is an optional [evals] extra; degrade if absent + from pydantic_evals.evaluators import Evaluator, EvaluatorContext + + @dataclass + class StructuralCitations(Evaluator): + """Hard gate — no fabricated (structurally-impossible) references.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + passed, _ = await check_structural_citations(ctx.output) + return passed + + @dataclass + class ReferenceLiveness(Evaluator): + """Graded — no dead-link (404/410) references.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + passed, _ = await check_reference_liveness(ctx.output) + return passed + + @dataclass + class CveIds(Evaluator): + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_cve_ids(ctx.output, ctx.expected_output or {})[0] + + @dataclass + class CvssWithin(Evaluator): + tol: float = 0.5 + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_cvss_within(ctx.output, ctx.expected_output or {}, tol=self.tol)[0] + + @dataclass + class NoJargonTitle(Evaluator): + async def evaluate(self, ctx: EvaluatorContext) -> bool: + return check_no_jargon_title(ctx.output)[0] + + @dataclass + class Abstention(Evaluator): + """Only meaningful on cases flagged ``abstain`` in metadata.""" + + async def evaluate(self, ctx: EvaluatorContext) -> bool: + if not (ctx.metadata or {}).get("abstain"): + return True + return check_abstention(ctx.output)[0] + + _HAS_PYDANTIC_EVALS = True +except ImportError: # pragma: no cover + _HAS_PYDANTIC_EVALS = False + + +__all__ = [ + "ReferenceAssessment", + "assess_references", + "check_abstention", + "check_cve_ids", + "check_cvss_within", + "check_no_jargon_title", + "check_reference_liveness", + "check_structural_citations", +] diff --git a/backend/cliff/evals/models.py b/backend/cliff/evals/models.py new file mode 100644 index 00000000..ed96cdb6 --- /dev/null +++ b/backend/cliff/evals/models.py @@ -0,0 +1,59 @@ +"""Eval model + provider selection (ADR-0050). + +Lives in the public ``cliff.evals`` package (not ``tests/``) so the in-repo +live test AND the private ``cliff-os/eval`` runner share ONE selection policy — +otherwise the two lanes can silently run different models and mask each other's +regressions. + +``select_eval_model`` returns the cheap-but-capable model for whichever provider +is configured, or ``None`` when nothing runnable is available. The live-test skip +gate is exactly "is there a runnable model?" — so it can't diverge from what the +runner will actually use (the conftest-gate-vs-eval_model bug this replaces). +""" + +from __future__ import annotations + +import os + +# Cheap-capable default per provider, keyed by the env var that selects it. +# Order matters: prefer the production default (Anthropic haiku) first. +_PROVIDER_DEFAULTS: tuple[tuple[str, str], ...] = ( + ("ANTHROPIC_API_KEY", "anthropic/claude-haiku-4-5"), + ("OPENAI_API_KEY", "openai/gpt-4o-mini"), + ("OPENROUTER_API_KEY", "openrouter/anthropic/claude-haiku-4.5"), + ("GEMINI_API_KEY", "google/gemini-2.5-flash"), + # Ollama needs a locally-pulled model name we can't assume, so it is NOT + # auto-selected — set CLIFF_EVAL_MODEL=ollama/ explicitly. +) + + +def harvest_env(source: dict[str, str] | None = None) -> dict[str, str]: + """The provider credentials the model factory reads (``*_API_KEY`` / + ``*_BASE_URL``), harvested from the host env.""" + env = source if source is not None else os.environ + return {k: v for k, v in env.items() if k.endswith(("_API_KEY", "_BASE_URL"))} + + +def select_eval_model(source: dict[str, str] | None = None) -> str | None: + """The model the eval should run, or ``None`` if nothing is runnable. + + ``CLIFF_EVAL_MODEL`` overrides (the operator owns matching the key); else the + first configured provider's cheap default. Match the production default + (Anthropic haiku) so the eval predicts customer experience. + """ + env = source if source is not None else os.environ + if override := env.get("CLIFF_EVAL_MODEL"): + return override + for key_var, model in _PROVIDER_DEFAULTS: + if env.get(key_var): + return model + return None + + +def eval_runnable(source: dict[str, str] | None = None) -> bool: + """True iff a live eval can actually run (a model resolves). The live-test + skip gate — kept identical to ``select_eval_model`` so they never diverge.""" + return select_eval_model(source) is not None + + +__all__ = ["eval_runnable", "harvest_env", "select_eval_model"] diff --git a/backend/cliff/evals/pricing.py b/backend/cliff/evals/pricing.py new file mode 100644 index 00000000..7a30a472 --- /dev/null +++ b/backend/cliff/evals/pricing.py @@ -0,0 +1,46 @@ +"""Approximate per-model token pricing for eval budget enforcement (ADR-0050 §4). + +USD per 1M tokens, ``(input, output)`` — provider list prices, approximate. Used +to turn token usage into a ``$`` estimate so the runner can enforce ``max_usd``. +The **token + duration caps are the reliable hard limits**; the ``$`` estimate is +best-effort (returns ``None`` for an unpriced model, and the runner then skips +the ``$`` check rather than guessing). Update as prices change / models are added. +""" + +from __future__ import annotations + +# Match is by substring against the model id (e.g. "openrouter/anthropic/ +# claude-haiku-4.5" matches "claude-haiku-4.5"). +_PRICE_USD_PER_MTOK: dict[str, tuple[float, float]] = { + "claude-haiku-4-5": (1.0, 5.0), + "claude-haiku-4.5": (1.0, 5.0), + "claude-sonnet-4": (3.0, 15.0), + "gpt-4o-mini": (0.15, 0.60), + "gpt-5": (1.25, 10.0), + "gemini-2.5-flash": (0.30, 2.50), +} + + +def _match(model_id: str | None) -> str | None: + """Longest matching key wins, so a model with its OWN key isn't mispriced + against a shorter prefix. A sub-model NOT in the table still falls back to + its nearest priced sibling (e.g. ``gpt-5-mini`` → ``gpt-5``) — an + over-estimate, which is the safe direction for a best-effort budget guard. + Add the sub-model's own key when an exact price matters.""" + m = (model_id or "").lower() + candidates = [k for k in _PRICE_USD_PER_MTOK if k in m] + return max(candidates, key=len) if candidates else None + + +def estimate_cost_usd( + model_id: str | None, input_tokens: int, output_tokens: int +) -> float | None: + """USD estimate for a single run, or ``None`` if the model isn't priced.""" + key = _match(model_id) + if key is None: + return None + in_price, out_price = _PRICE_USD_PER_MTOK[key] + return input_tokens / 1e6 * in_price + output_tokens / 1e6 * out_price + + +__all__ = ["estimate_cost_usd"] diff --git a/backend/cliff/evals/registry.py b/backend/cliff/evals/registry.py new file mode 100644 index 00000000..d6adcd42 --- /dev/null +++ b/backend/cliff/evals/registry.py @@ -0,0 +1,100 @@ +"""Per-agent eval registry (ADR-0050 §1). + +One ``AgentEvalSpec`` per eval target — the single source of truth for what +each agent supports, its budget, and how to build it. ``supported_assertions`` +documents which assertion families are meaningful for the agent; per-case +datasets don't yet declare assertions (the per-agent runner applies a fixed +set), so it is descriptive, not yet runner-enforced — that validation lands +when cases carry explicit per-case assertions. + +Only ``finding_enricher`` is wired today (ADR-0050 rollout §7: highest-risk +first). Add entries as each agent's eval lands. +""" + +# No ``from __future__ import annotations`` here: Pydantic resolves these field +# annotations eagerly at class definition, so ``Callable`` / ``Model`` / ``Agent`` +# are genuinely runtime imports (and ruff TC-flags them only under future +# annotations). +from collections.abc import Callable + +from pydantic import BaseModel, ConfigDict +from pydantic_ai import Agent +from pydantic_ai.models import Model + +from cliff.agents.runtime.finding_enricher import build_agent as _build_enricher +from cliff.agents.schemas import EnrichmentOutput + + +class BudgetSpec(BaseModel): + """Budget ceilings, enforced by the runner (ADR-0050 §4). The token + time + caps are reliable hard limits; ``*_usd`` is best-effort (skipped when the + model isn't in the pricing table). A breach fails the eval run.""" + + model_config = ConfigDict(frozen=True) + + # Per case. + max_usd: float | None = None + max_tokens: int | None = None + max_duration_s: float | None = None + # Per run (whole dataset) — the runaway-bill stop. + max_run_usd: float | None = None + max_run_tokens: int | None = None + + +class AgentEvalSpec(BaseModel): + # ``build_agent`` (a callable) and ``output_type`` (a class) aren't Pydantic + # types, so allow arbitrary types; frozen for an immutable registry. + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + name: str + build_agent: Callable[[Model], Agent] + output_type: type[BaseModel] + abstention_required: bool + supported_assertions: frozenset[str] + budget: BudgetSpec + default_model: str | None = None + + +_REGISTRY: dict[str, AgentEvalSpec] = { + "finding_enricher": AgentEvalSpec( + name="finding_enricher", + build_agent=_build_enricher, + output_type=EnrichmentOutput, + abstention_required=True, + supported_assertions=frozenset( + { + "citation_liveness", + "cve_ids", + "cvss_within", + "no_jargon_title", + "abstention", + } + ), + budget=BudgetSpec( + max_usd=0.03, + max_tokens=8000, + max_duration_s=60.0, + max_run_usd=0.50, + max_run_tokens=120_000, + ), + # Match the production default so the eval predicts customer experience; + # used by the adapter (and USD pricing) when a caller omits model_id. + default_model="anthropic/claude-haiku-4-5", + ), +} + + +def get_spec(name: str) -> AgentEvalSpec: + try: + return _REGISTRY[name] + except KeyError: + raise KeyError( + f"No eval spec for {name!r}. Registered: {sorted(_REGISTRY)}" + ) from None + + +def all_specs() -> list[AgentEvalSpec]: + return list(_REGISTRY.values()) + + +__all__ = ["AgentEvalSpec", "BudgetSpec", "all_specs", "get_spec"] diff --git a/backend/cliff/evals/runners.py b/backend/cliff/evals/runners.py new file mode 100644 index 00000000..4e17233f --- /dev/null +++ b/backend/cliff/evals/runners.py @@ -0,0 +1,183 @@ +"""Reusable eval run loops (ADR-0050). + +The orchestration lives here — in the public ``cliff`` package — so it can be +driven from BOTH the in-repo live test (public synthetic sample) AND the +private ``cliff-os/eval`` project (real/confidential datasets), which depends +on ``cliff`` and only supplies its own dataset dir. Same scorer, two data +sources (ADR-0050 hybrid). + +Agent-specific for now (only ``finding_enricher`` is wired). Generalize into a +registry-driven loop when the second agent lands. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from cliff.evals.adapter import run_agent_measured +from cliff.evals.evaluators import ( + assess_references, + check_abstention, + check_cve_ids, + check_cvss_within, + check_no_jargon_title, +) +from cliff.evals.pricing import estimate_cost_usd +from cliff.evals.registry import get_spec + +if TYPE_CHECKING: + from cliff.evals.cases import EvalCase + + +@dataclass +class EvalRunResult: + agent: str + n_cases: int + graded_floor: float + hard_failures: list[str] = field(default_factory=list) + budget_failures: list[str] = field(default_factory=list) + graded_rates: dict[str, float] = field(default_factory=dict) + # Advisory metrics are reported + tracked but do NOT gate pass/fail — e.g. + # reference_liveness, since production strips dead links so they never reach + # the user. Only structural fabrication (the hard gate) is user-harmful. + advisory: set[str] = field(default_factory=set) + total_tokens: int = 0 + est_cost_usd: float | None = None + + @property + def passed(self) -> bool: + return ( + not self.hard_failures + and not self.budget_failures + and all( + rate >= self.graded_floor + for metric, rate in self.graded_rates.items() + if metric not in self.advisory + ) + ) + + def report(self) -> str: + cost = f"~${self.est_cost_usd:.4f}" if self.est_cost_usd is not None else "$?" + lines = [ + f"{self.agent}: {self.n_cases} cases — {'PASS' if self.passed else 'FAIL'}" + f" ({self.total_tokens:,} tok, {cost})" + ] + for metric, rate in sorted(self.graded_rates.items()): + if metric in self.advisory: + lines.append(f" advis. {metric:18s} {rate:.0%} (tracked, non-gating)") + continue + flag = "" if rate >= self.graded_floor else f" ⛔ < {self.graded_floor:.0%}" + lines.append(f" graded {metric:18s} {rate:.0%}{flag}") + for hf in self.hard_failures: + lines.append(f" ⛔ HARD {hf}") + for bf in self.budget_failures: + lines.append(f" ⛔ BUDGET {bf}") + return "\n".join(lines) + + +async def run_enricher_eval( + cases: list[EvalCase], + *, + env: dict[str, str], + model_id: str | None, + graded_floor: float = 0.6, +) -> EvalRunResult: + """Run the real finding_enricher over *cases* and score it. + + Hard gates (zero tolerance, collected into ``hard_failures``): structural + citation fabrication on every case, abstention on every ``abstain`` case. + Graded metrics (dataset-wide pass rate vs ``graded_floor``): cve_ids, + cvss_within, no_jargon_title, reference_liveness. + """ + if not cases: + raise ValueError( + "run_enricher_eval got 0 cases — an empty dataset must fail, not " + "silently report PASS. Check the dataset path / tier filter." + ) + + spec = get_spec("finding_enricher") + budget = spec.budget + result = EvalRunResult( + agent=spec.name, + n_cases=len(cases), + graded_floor=graded_floor, + # Dead-link guesses are stripped by production before the user sees + # them, so reference_liveness is a model-quality signal, not a gate. + advisory={"reference_liveness"}, + ) + graded: dict[str, list[bool]] = { + "cve_ids": [], + "cvss_within": [], + "no_jargon_title": [], + "reference_liveness": [], + } + # Price against the model the adapter actually runs (it falls back to + # spec.default_model when model_id is None), so a None id can't silently + # skip the USD caps. + pricing_model_id = model_id or spec.default_model + run_cost = 0.0 + run_cost_known = True + + for case in cases: + run = await run_agent_measured(spec, case.finding, env=env, model_id=model_id) + out = run.output + + # Per-case budget (ADR-0050 §4): tokens + duration are hard; $ is + # best-effort (skipped when the model isn't priced). + if budget.max_tokens is not None and run.total_tokens > budget.max_tokens: + result.budget_failures.append( + f"{case.id}: {run.total_tokens:,} tok > {budget.max_tokens:,} cap" + ) + if budget.max_duration_s is not None and run.duration_s > budget.max_duration_s: + result.budget_failures.append( + f"{case.id}: {run.duration_s:.0f}s > {budget.max_duration_s:.0f}s cap" + ) + case_cost = estimate_cost_usd( + pricing_model_id, run.input_tokens, run.output_tokens + ) + if case_cost is None: + run_cost_known = False + else: + run_cost += case_cost + if budget.max_usd is not None and case_cost > budget.max_usd: + result.budget_failures.append( + f"{case.id}: ~${case_cost:.4f} > ${budget.max_usd:.4f} cap" + ) + result.total_tokens += run.total_tokens + + refs = await assess_references(out) + if refs.structural: + bad = "; ".join(f"{u} ({why})" for u, why in refs.structural) + result.hard_failures.append(f"{case.id}: structural_citations — {bad}") + if case.abstain: + ok, reason = check_abstention(out) + if not ok: + result.hard_failures.append(f"{case.id}: abstention — {reason}") + + expected = case.expected.as_dict() + graded["cve_ids"].append(check_cve_ids(out, expected)[0]) + graded["cvss_within"].append(check_cvss_within(out, expected)[0]) + graded["no_jargon_title"].append(check_no_jargon_title(out)[0]) + graded["reference_liveness"].append(not refs.network) + + # Per-run budget (the runaway-bill stop). + if budget.max_run_tokens is not None and result.total_tokens > budget.max_run_tokens: + result.budget_failures.append( + f"run: {result.total_tokens:,} tok > {budget.max_run_tokens:,} cap" + ) + if run_cost_known: + result.est_cost_usd = run_cost + if budget.max_run_usd is not None and run_cost > budget.max_run_usd: + result.budget_failures.append( + f"run: ~${run_cost:.4f} > ${budget.max_run_usd:.4f} cap" + ) + + result.graded_rates = { + metric: (sum(vals) / len(vals) if vals else 1.0) + for metric, vals in graded.items() + } + return result + + +__all__ = ["EvalRunResult", "run_enricher_eval"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c94aa4fe..0cd36959 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -49,3 +49,10 @@ markers = [ "docker: install smoke tests that boot the published Docker image (require docker daemon)", "integration: cross-cutting integration tests against an in-memory DB (no external deps, run by default)", ] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["cliff"] diff --git a/backend/tests/agents/conftest.py b/backend/tests/agents/conftest.py index 00a496b5..8eb4bfa5 100644 --- a/backend/tests/agents/conftest.py +++ b/backend/tests/agents/conftest.py @@ -8,21 +8,26 @@ from __future__ import annotations -import os - import pytest -_api_key_set = bool( - os.environ.get("OPENAI_API_KEY") or os.environ.get("ANTHROPIC_API_KEY") -) +from cliff.evals.models import eval_runnable +# Gate live-LLM tests on whether a model actually resolves (CLIFF_EVAL_MODEL +# override or a configured provider) — derived from the SAME policy the runner +# uses, so the gate can't say "runnable" while eval_model picks a model with no +# key (e.g. an Ollama-only host that would otherwise hard-error, not skip). _skip_no_key = pytest.mark.skipif( - not _api_key_set, reason="No LLM API key set (OPENAI_API_KEY or ANTHROPIC_API_KEY)" + not eval_runnable(), + reason="No runnable eval model (set a provider key or CLIFF_EVAL_MODEL)", ) # Files whose every test hits a real LLM (the live evals). Everything else # under tests/agents/ runs against FunctionModel/TestModel and stays in CI. -_LIVE_LLM_FILES = ("test_normalizer_agent", "test_plain_description_eval") +_LIVE_LLM_FILES = ( + "test_normalizer_agent", + "test_plain_description_eval", + "test_evals_finding_enricher", +) def pytest_collection_modifyitems(items): diff --git a/backend/tests/agents/eval/README.md b/backend/tests/agents/eval/README.md new file mode 100644 index 00000000..c374eaa3 --- /dev/null +++ b/backend/tests/agents/eval/README.md @@ -0,0 +1,26 @@ +# Agent eval datasets — public synthetic samples + +These JSONL files are the **public, synthetic sample** datasets for the agent +eval harness (ADR-0050). They exist so the eval is: + +- **runnable in open source** (a contributor can run the live lane), and +- **deterministic in CI** (the harness tests load them). + +They are intentionally small and contain **no proprietary or confidential +data** — only well-known public CVEs and hand-authored synthetic findings. + +## The real datasets live elsewhere (private) + +The proprietary / confidential golden sets — and any customer-derived data — +live in the **private `cliff-os/eval/`** project, never in this public repo +(ADR-0050 "harness public, data private"). + +The harness reads its dataset directory from `CLIFF_EVAL_DATASET_DIR`: + +| Run | `CLIFF_EVAL_DATASET_DIR` | Dataset | +|-----|--------------------------|---------| +| public CI / OSS | _(unset)_ | this directory (synthetic sample) | +| private eval pipeline | `cliff-os/eval/datasets` | the real golden sets | + +So the *scoring logic* is open and the *data* stays private. To add a public +sample case, append a line here; real/sensitive cases go in `cliff-os/eval`. diff --git a/backend/tests/agents/eval/finding_enricher.jsonl b/backend/tests/agents/eval/finding_enricher.jsonl new file mode 100644 index 00000000..a6612457 --- /dev/null +++ b/backend/tests/agents/eval/finding_enricher.jsonl @@ -0,0 +1,6 @@ +{"id": "log4shell", "tier": "live", "edge_case": "well_known_cve", "abstain": false, "finding": {"title": "Remote code execution via JNDI lookup in Apache Log4j", "description": "The application bundles Apache Log4j 2.14.1. A crafted log message containing a ${jndi:ldap://...} lookup string allows remote code execution.", "raw_severity": "critical", "source_type": "trivy", "source_id": "CVE-2021-44228", "asset_label": "api-service"}, "expected": {"cve_ids": ["CVE-2021-44228"], "cvss_min": 9.0}} +{"id": "lodash-prototype-pollution", "tier": "live", "edge_case": "dependency_cve", "abstain": false, "finding": {"title": "Prototype Pollution in lodash", "description": "lodash 4.17.20 is vulnerable to prototype pollution via the set and setWith functions when used with attacker-controlled property paths.", "raw_severity": "high", "source_type": "snyk", "source_id": "SNYK-JS-LODASH-1018905", "asset_label": "web-frontend"}, "expected": {"cve_ids": ["CVE-2020-8203"], "cvss_min": 5.5, "cvss_max": 8.5}} +{"id": "jargon-title-strip", "tier": "live", "edge_case": "scanner_jargon_title", "abstain": false, "finding": {"title": "SNYK-JS-MINIMIST-559764: Prototype Pollution", "description": "minimist before 1.2.3 is vulnerable to prototype pollution. The title carries the raw Snyk rule id which must be stripped.", "raw_severity": "medium", "source_type": "snyk", "source_id": "SNYK-JS-MINIMIST-559764", "asset_label": "cli-tool"}, "expected": {"cvss_min": 4.0, "cvss_max": 9.9}} +{"id": "semgrep-no-cve-eval", "tier": "live", "edge_case": "no_cve_code_finding", "abstain": true, "finding": {"title": "Use of eval() on user-controlled input", "description": "The handler passes request.body directly to eval(), allowing arbitrary code execution. This is a first-party code defect found by static analysis; it has no CVE.", "raw_severity": "high", "source_type": "semgrep", "source_id": "javascript.lang.security.detect-eval-with-expression", "asset_label": "orders-api"}, "expected": {"cve_ids": []}} +{"id": "posture-no-cve-missing-header", "tier": "live", "edge_case": "no_cve_posture_finding", "abstain": true, "finding": {"title": "Missing Content-Security-Policy header", "description": "Responses from the public site do not set a Content-Security-Policy header. A configuration/posture issue, not a tracked CVE.", "raw_severity": "low", "source_type": "posture", "source_id": "missing_security_header", "asset_label": "marketing-site"}, "expected": {"cve_ids": []}} +{"id": "obscure-pkg-fabrication-bait", "tier": "live", "edge_case": "fabrication_bait", "abstain": true, "finding": {"title": "Vulnerability in internal-telemetry-shim 0.3.1", "description": "A dependency scan flagged internal-telemetry-shim@0.3.1 (a private/obscure package) as potentially vulnerable, but provided no advisory id, CVE, or details. The enricher must not invent a GHSA id, CVE, CVSS, or advisory URL.", "raw_severity": "medium", "source_type": "snyk", "source_id": "", "asset_label": "telemetry"}, "expected": {"cve_ids": []}} diff --git a/backend/tests/agents/eval_utils.py b/backend/tests/agents/eval_utils.py index b96d3e2e..b2797763 100644 --- a/backend/tests/agents/eval_utils.py +++ b/backend/tests/agents/eval_utils.py @@ -1,30 +1,16 @@ -"""Shared setup for the real-LLM agent evals (IMPL-0022 PR #3b). +"""Shared setup for the real-LLM agent evals. -The two live-LLM test modules — ``test_normalizer_agent`` and -``test_plain_description_eval`` — both need the same provider env + a -capable-but-cheap model id. Keeping that selection policy here means it -can't drift between the two files. Both modules are skip-gated on an API -key being present (see ``conftest.py``). +Thin re-export of the canonical selection policy in ``cliff.evals.models`` — +which lives in the public package (not ``tests/``) so the in-repo live tests AND +the private ``cliff-os/eval`` runner share ONE policy and can't drift. Live test +modules are skip-gated on a runnable model being available (see ``conftest.py``). """ from __future__ import annotations -import os +from cliff.evals.models import harvest_env, select_eval_model -# Provider credentials for the app-level normalizer, harvested from the host -# env (the same shape the running daemon resolves into a workspace). -LLM_ENV = { - k: v for k, v in os.environ.items() if k.endswith(("_API_KEY", "_BASE_URL")) -} - - -def eval_model() -> str: - """Capable, cheap model for the real-LLM eval (override: ``CLIFF_EVAL_MODEL``).""" - if override := os.environ.get("CLIFF_EVAL_MODEL"): - return override - if os.environ.get("ANTHROPIC_API_KEY"): - return "anthropic/claude-haiku-4-5" - return "openai/gpt-4o-mini" - - -LLM_MODEL = eval_model() +LLM_ENV = harvest_env() +# Resolved at import; only *used* by tests that are skip-gated on a runnable +# model, so the last-resort fallback is just to keep this non-None at import. +LLM_MODEL = select_eval_model() or "openai/gpt-4o-mini" diff --git a/backend/tests/agents/test_evals_finding_enricher.py b/backend/tests/agents/test_evals_finding_enricher.py new file mode 100644 index 00000000..1fd4b6c2 --- /dev/null +++ b/backend/tests/agents/test_evals_finding_enricher.py @@ -0,0 +1,30 @@ +"""Live lane for the finding_enricher eval (ADR-0050 §5, Lane 2). + +Runs the REAL enricher over the active dataset (the public synthetic sample by +default; the private ``cliff-os/eval`` datasets when ``CLIFF_EVAL_DATASET_DIR`` +is set — ADR-0050 hybrid) and scores it via the shared ``run_enricher_eval`` +loop. Skipped automatically when no API key is set (see ``conftest.py`` — this +file is in ``_LIVE_LLM_FILES``); never masks the deterministic CI lane. + +Hard gates (zero tolerance): structural citation fabrication on every case + +abstention on every ``abstain`` case. Graded metrics (cve_ids / cvss_within / +no_jargon_title / reference_liveness) must clear a dataset-wide floor. + +Budget: a handful of LLM calls + HTTP HEADs, well under the registry ceiling. +""" + +from __future__ import annotations + +from cliff.evals import load_cases, run_enricher_eval +from tests.agents.eval_utils import LLM_ENV as _LLM_ENV +from tests.agents.eval_utils import LLM_MODEL as _LLM_MODEL + + +async def test_finding_enricher_eval(): + cases = load_cases("finding_enricher", tier="live") + assert cases, "no live cases in the active finding_enricher dataset" + + result = await run_enricher_eval( + cases, env=_LLM_ENV, model_id=_LLM_MODEL, graded_floor=0.6 + ) + assert result.passed, "\n" + result.report() diff --git a/backend/tests/agents/test_evals_harness.py b/backend/tests/agents/test_evals_harness.py new file mode 100644 index 00000000..693670ea --- /dev/null +++ b/backend/tests/agents/test_evals_harness.py @@ -0,0 +1,144 @@ +"""CI lane for the eval harness (ADR-0050 §5, Lane 1). + +Deterministic, no API key, no real network. Proves the finding_enricher +evaluators + adapter + registry/dataset wiring are correct — so a green CI +means the harness itself won't silently mis-score the live lane. +""" + +from __future__ import annotations + +from pydantic_ai.models.test import TestModel + +from cliff.agents.schemas import EnrichmentOutput +from cliff.evals import get_spec, load_cases, run_agent +from cliff.evals.evaluators import ( + check_abstention, + check_cve_ids, + check_cvss_within, + check_no_jargon_title, + check_reference_liveness, + check_structural_citations, +) + +# --- citation_liveness (the fail-closed gate) -------------------------------- + + +class _FakeResp: + def __init__(self, status_code: int) -> None: + self.status_code = status_code + + +class _FakeHttp: + """Stand-in httpx.AsyncClient: every probe returns *status*.""" + + def __init__(self, status: int = 200) -> None: + self._status = status + + async def get(self, url: str, **_kw: object) -> _FakeResp: + return _FakeResp(self._status) + + async def aclose(self) -> None: # pragma: no cover - never owns the client + pass + + +async def test_structural_gate_fails_on_fabricated_github_sha(): + """A garbled commit SHA is structurally bogus → dropped in the no-network + sanitize pass → the HARD gate FAILS (the agent fabricated a citation).""" + out = EnrichmentOutput( + normalized_title="x", + references=["https://github.com/foo/bar/commit/zzzzzzz"], + ) + passed, reason = await check_structural_citations(out) + assert passed is False + assert "fabricated" in reason + + +async def test_liveness_graded_separates_dead_link_from_fabrication(): + """A plausible URL that 404s is a GRADED dead-link, NOT a structural-gate + failure — the key distinction the live eval taught us.""" + out = EnrichmentOutput( + normalized_title="x", references=["https://example.com/moved-page"] + ) + # structural gate passes (the URL is well-formed)... + structural_ok, _ = await check_structural_citations(out, http=_FakeHttp(404)) + assert structural_ok is True + # ...but the graded liveness check flags the 404. + live_ok, reason = await check_reference_liveness(out, http=_FakeHttp(404)) + assert live_ok is False + assert "dead links" in reason + + +async def test_both_gates_pass_for_live_and_empty(): + live = EnrichmentOutput( + normalized_title="x", + references=["https://nvd.nist.gov/vuln/detail/CVE-2021-44228"], + ) + assert (await check_structural_citations(live, http=_FakeHttp(200)))[0] is True + assert (await check_reference_liveness(live, http=_FakeHttp(200)))[0] is True + empty = EnrichmentOutput(normalized_title="x", references=[]) + assert (await check_structural_citations(empty))[0] is True + assert (await check_reference_liveness(empty))[0] is True + + +# --- deterministic field evaluators ------------------------------------------ + + +def test_cve_ids_set_match_case_insensitive(): + out = EnrichmentOutput(normalized_title="x", cve_ids=["cve-2021-44228"]) + assert check_cve_ids(out, {"cve_ids": ["CVE-2021-44228"]})[0] is True + assert check_cve_ids(out, {"cve_ids": ["CVE-2020-8203"]})[0] is False + + +def test_cvss_within_tolerance_and_range_and_null(): + out = EnrichmentOutput(normalized_title="x", cvss_score=9.8) + assert check_cvss_within(out, {"cvss_score": 10.0})[0] is True # within ±0.5 + assert check_cvss_within(out, {"cvss_score": 7.0})[0] is False + assert check_cvss_within(out, {"cvss_min": 9.0, "cvss_max": 10.0})[0] is True + null_out = EnrichmentOutput(normalized_title="x", cvss_score=None) + assert check_cvss_within(null_out, {"cvss_score": None})[0] is True + + +def test_no_jargon_title_flags_scanner_ids(): + clean = EnrichmentOutput(normalized_title="Prototype pollution in lodash") + assert check_no_jargon_title(clean)[0] is True + jargon = EnrichmentOutput( + normalized_title="SNYK-JS-LODASH-1018905: Prototype Pollution" + ) + assert check_no_jargon_title(jargon)[0] is False + + +def test_abstention_rejects_invented_cve_or_cvss(): + abstained = EnrichmentOutput(normalized_title="x", cve_ids=[], cvss_score=None) + assert check_abstention(abstained)[0] is True + invented_cve = EnrichmentOutput(normalized_title="x", cve_ids=["CVE-2026-99999"]) + assert check_abstention(invented_cve)[0] is False + invented_cvss = EnrichmentOutput(normalized_title="x", cvss_score=7.5) + assert check_abstention(invented_cvss)[0] is False + + +# --- adapter + registry + dataset wiring ------------------------------------- + + +async def test_adapter_runs_enricher_via_testmodel(): + """run_agent builds + drives the real enricher agent; TestModel synthesises + a schema-valid EnrichmentOutput so the wiring is exercised without a key.""" + spec = get_spec("finding_enricher") + out = await run_agent(spec, {"title": "x", "description": "y"}, model=TestModel()) + assert isinstance(out, EnrichmentOutput) + + +def test_dataset_loads_and_assertions_are_supported(): + spec = get_spec("finding_enricher") + cases = load_cases("finding_enricher") + assert len(cases) >= 5 + assert {c.id for c in cases} >= {"log4shell", "semgrep-no-cve-eval"} + # every abstain case declares an empty expected cve_ids (abstention contract) + for c in cases: + if c.abstain: + assert c.expected.cve_ids == [] + # registry advertises exactly the assertion families this agent's evaluators + # implement (ADR-0050 §1: dataset assertions ⊆ supported_assertions) + assert spec.supported_assertions == frozenset( + {"citation_liveness", "cve_ids", "cvss_within", "no_jargon_title", "abstention"} + ) + assert spec.abstention_required is True diff --git a/backend/uv.lock b/backend/uv.lock index c1d52d57..959f2307 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -503,7 +503,7 @@ wheels = [ [[package]] name = "cliff" version = "0.2.1" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "aiosqlite" }, { name = "cryptography" },