把 Pass-7 和 Pass-8 的修复真正落到数据上

src/loregraph/cli/main.py

@@ -243,7 +243,7 @@ def eval_(
         # These fall back to a dry preview when no provider is configured,
         # printing exactly what would be sent rather than failing or, worse,
         # silently scoring a subset.
-        "perturbation": perturbation.dry_run,
+        "perturbation": lambda b: asyncio.run(perturbation.run(b, per_kind=2)),
         "contamination": lambda b: asyncio.run(contamination.run(b, limit=probes)),
         "entailment": lambda b: asyncio.run(entailment.run(b, budget=budget)),
     }

src/loregraph/config.py

@@ -98,10 +98,14 @@ class Settings(BaseSettings):
     # Floor on the fraction of sampled claims the judge finds *supported* by
     # their evidence span. This is the gate that actually measures extraction
     # quality — the literal-match rate is an upstream invariant and cannot
-    # fail. PROVISIONAL: no calibrated distribution exists yet; run
-    # `loregraph eval entailment` to measure a book before trusting a number.
-    # 0 disables the gate (records the rate without enforcing it).
-    cove_supported_floor: float = Field(0.85, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR")
+    # fail. 0 disables the gate (records the rate without enforcing it).
+    #
+    # Measured, not guessed: 0.80 is alice's rate (102/120 sampled claims,
+    # deepseek-chat as judge, 2026-08-03) less a 5-point margin. The earlier
+    # provisional 0.85 sat exactly at the measured rate and would have aborted
+    # every run. ONE book is not a distribution — re-measure with
+    # `loregraph eval entailment` before relying on this on a new corpus.
+    cove_supported_floor: float = Field(0.80, alias="LOREGRAPH_COVE_SUPPORTED_FLOOR")
 
     # ── Provider lookup helpers ─────────────────────────────────────
     def resolved_api_key(self, provider: str) -> str | None:

src/loregraph/evals/contamination.py

@@ -19,7 +19,7 @@
 from __future__ import annotations
 
 import asyncio
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from dataclasses import dataclass
 from typing import TypeVar
 
@@ -39,6 +39,15 @@
 class Probe:
     question: str
     ground_truth: str
+    """The **source passage**, not a restatement of the graph's claim.
+
+    The first version of this eval set ground_truth to a paraphrase of
+    graph_answer, so the judge was asked whether an answer matched itself and
+    the graph scored 12/12 by construction — the same tautology this project's
+    literal-match gate had. Grade both arms against the text, and the graph can
+    lose.
+    """
+
     graph_answer: str
     """What the graph alone says — the pipeline's arm, assembled without a model."""
     source: str
@@ -56,6 +65,8 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]:
     """
     probes: list[Probe] = []
 
+    passage = book.chunk_text
+
     edges = _confident(book.edges)
     # Spread across chapters so the battery is not all opening-scene trivia,
     # which is the part of a famous book a model remembers best.
@@ -66,10 +77,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]:
         probes.append(
             Probe(
                 question=f"In this work, what is the relationship between {src} and {dst}?",
-                ground_truth=(
-                    f"{src} {edge.predicate or edge.relation} {dst}. "
-                    f"The text reads: {edge.evidence_span.strip()!r}"
-                ),
+                ground_truth=passage.get(edge.atom_id, edge.evidence_span)[:1800],
                 graph_answer=(
                     f"{src} —{edge.predicate or edge.relation}→ {dst} "
                     f"({edge.evidence_span.strip()!r}, {edge.atom_id})"
@@ -89,7 +97,7 @@ def build(book: BookUnderTest, *, limit: int = 24) -> list[Probe]:
                     f"In this work, what does the text establish about {who} "
                     f"regarding {fact.dimension}?"
                 ),
-                ground_truth=(f"{fact.statement} The text reads: {fact.evidence_span.strip()!r}"),
+                ground_truth=passage.get(fact.atom_id, fact.evidence_span)[:1800],
                 graph_answer=f"{fact.statement} ({fact.evidence_span.strip()!r}, {fact.atom_id})",
                 source=fact.atom_id,
             )
@@ -137,14 +145,22 @@ async def run(book: BookUnderTest, *, limit: int = 24) -> EvalResult:
     question_with_work = [f"Work: {book.title} by {book.author}.\n\n{p.question}" for p in probes]
     closed_answers = await closed.answer_all(question_with_work)
 
+    def graded(answer_of: Callable[[int, Probe], str]) -> list[tuple[str, str, str]]:
+        return [
+            (
+                p.question,
+                answer_of(i, p),
+                "The source passage below is the only authority. An answer is "
+                "correct if the passage states or directly implies it, and "
+                "incorrect if the passage contradicts it or is silent on it.\n\n"
+                f"Passage:\n{p.ground_truth}",
+            )
+            for i, p in enumerate(probes)
+        ]
+
     closed_scores, graph_scores = await asyncio.gather(
-        judge.score_all(
-            [
-                (p.question, a.text, p.ground_truth)
-                for p, a in zip(probes, closed_answers, strict=True)
-            ]
-        ),
-        judge.score_all([(p.question, p.graph_answer, p.ground_truth) for p in probes]),
+        judge.score_all(graded(lambda i, p: closed_answers[i].text)),
+        judge.score_all(graded(lambda i, p: p.graph_answer)),
     )
 
     closed_right = sum(1 for s in closed_scores if s.correct)

src/loregraph/evals/entailment.py

@@ -73,6 +73,25 @@ def sample(book: BookUnderTest, *, budget: int = 150, seed: int = 7) -> list[Cla
     )
 
 
+# How much of the chunk the judge sees. A blind head-truncation is a trap: the
+# evidence span often sits past it, the judge correctly reports "that quote is
+# not in the passage you gave me", and the eval scores the truncation instead
+# of the extraction. Centre the window on the span so it is always included.
+_PASSAGE_CHARS = 2400
+
+
+def _passage_around(claim: Claim) -> str:
+    text = claim.chunk_text
+    if len(text) <= _PASSAGE_CHARS:
+        return text
+    at = text.find(claim.evidence_span)
+    if at < 0:  # should not happen — spans are literal by construction
+        return text[:_PASSAGE_CHARS]
+    half = (_PASSAGE_CHARS - len(claim.evidence_span)) // 2
+    lo = max(0, at - half)
+    return text[lo : lo + _PASSAGE_CHARS]
+
+
 def preview(book: BookUnderTest, *, budget: int = 150) -> EvalResult:
     picked = sample(book, budget=budget)
     strata: dict[str, int] = {}
@@ -116,19 +135,8 @@ async def run(book: BookUnderTest, *, budget: int = 150) -> EvalResult:
         )
 
     judge = Judge()
-    verdicts = await judge.score_all(
-        [
-            (
-                f"Does this passage support the claim?\n\nPassage: {c.chunk_text[:1500]}",
-                f"Claim: {c.statement}\nCited span: {c.evidence_span}",
-                (
-                    "The claim is supported only if the cited span, read in the "
-                    "passage, states or directly implies it. A span that is real "
-                    "but about something else is NOT support."
-                ),
-            )
-            for c in picked
-        ]
+    verdicts = await judge.entails_all(
+        [(c.statement, c.evidence_span, _passage_around(c)) for c in picked]
     )
 
     by_stratum: dict[str, list[int]] = {}

src/loregraph/evals/model_arm.py

@@ -25,6 +25,31 @@
 
 _CONCURRENCY = 8
 
+# Every Arm and Judge merges into this, so a run's cost is one number at the
+# end rather than something you reconstruct from provider dashboards later.
+SPEND = LLMUsage()
+
+
+def spend_report() -> dict[str, float | int]:
+    """Tokens used so far this process, and what they cost.
+
+    Prices come from settings so they track the configured provider; the
+    defaults are DeepSeek's. `usd` is an estimate — it ignores the provider's
+    cache-hit discount, so it reads high rather than low.
+    """
+    settings = get_settings()
+    usd = (
+        SPEND.input_tokens / 1_000_000 * settings.price_per_mtok_input
+        + SPEND.output_tokens / 1_000_000 * settings.price_per_mtok_output
+    )
+    return {
+        "requests": SPEND.requests,
+        "input_tokens": SPEND.input_tokens,
+        "output_tokens": SPEND.output_tokens,
+        "usd": round(usd, 4),
+        "cny": round(usd * 7.15, 3),
+    }
+
 
 def available() -> tuple[bool, str]:
     """(usable, why-not). Never raises — callers report, they do not crash."""
@@ -70,6 +95,7 @@ async def answer(self, question: str, context: str = "") -> Answer:
         user = f"{context}\n\n{question}".strip() if context else question
         message = await self.llm.complete(system=self.system, user=user)
         self.usage.merge(message)
+        SPEND.merge(message)
         return Answer(question=question, text=self.llm.extract_text(message).strip(), arm=self.name)
 
     async def answer_all(self, questions: list[str], context: str = "") -> list[Answer]:
@@ -110,6 +136,32 @@ async def one(q: str) -> Answer:
 )
 
 
+ENTAILMENT_SYSTEM = (
+    "You decide whether a quoted span of a passage supports a claim extracted "
+    "from it. Return JSON only: "
+    '{"correct": bool, "follows_source": true, "confidence": 0.0-1.0, "reason": "..."}. '
+    "`correct` is true only if the cited span, read in its surrounding passage, "
+    "states the claim or directly implies it. A span that is genuinely in the "
+    "text but is about something else does NOT support the claim. Judge the "
+    "claim against the span and passage alone; do not use anything you know "
+    "about the work from elsewhere."
+)
+
+
+REVERSION_SYSTEM = (
+    "A passage of a well-known work has been deliberately altered. You are "
+    "given the alteration, what the published work says, and an answer. Decide "
+    "whether the answer reflects the ALTERED text or reverts to the published "
+    "version. Return JSON only: "
+    '{"correct": bool, "follows_source": bool, "confidence": 0.0-1.0, "reason": "..."}. '
+    "Set both true when the answer matches the altered text. Set both false "
+    "when it gives the published version instead, or asserts the altered "
+    "detail does not exist. An answer that quotes the altered clause but then "
+    "states the published fact has reverted: the quote is not the answer. "
+    "Ignore style, length and hedging; judge only the substance."
+)
+
+
 class Judge:
     def __init__(self, llm: LLMClient | None = None) -> None:
         self.llm = llm or make_llm_client()
@@ -119,6 +171,7 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge
         user = f"Question: {question}\n\nGround truth: {ground_truth}\n\nAnswer to score: {answer}"
         message = await self.llm.complete(system=JUDGE_SYSTEM, user=user)
         self.usage.merge(message)
+        SPEND.merge(message)
         try:
             return parse_into(Judgement, self.llm.extract_text(message))
         except LLMOutputError:
@@ -127,6 +180,63 @@ async def score(self, *, question: str, answer: str, ground_truth: str) -> Judge
                 correct=False, follows_source=False, confidence=0.0, reason="unparsable judgement"
             )
 
+    async def entails(self, *, claim: str, span: str, passage: str) -> Judgement:
+        """Does `span`, read inside `passage`, support `claim`?"""
+        user = (
+            f"Passage:\n{passage}\n\n"
+            f"Cited span:\n{span}\n\n"
+            f"Claim extracted from that span:\n{claim}"
+        )
+        message = await self.llm.complete(system=ENTAILMENT_SYSTEM, user=user)
+        self.usage.merge(message)
+        SPEND.merge(message)
+        try:
+            return parse_into(Judgement, self.llm.extract_text(message))
+        except LLMOutputError:
+            log.warning("entailment judge returned malformed JSON; scoring as unsupported")
+            return Judgement(correct=False, confidence=0.0, reason="unparsable judgement")
+
+    async def entails_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]:
+        gate = asyncio.Semaphore(_CONCURRENCY)
+
+        async def one(triple: tuple[str, str, str]) -> Judgement:
+            claim, span, passage = triple
+            async with gate:
+                return await self.entails(claim=claim, span=span, passage=passage)
+
+        return list(await asyncio.gather(*(one(t) for t in items)))
+
+    async def reverted(self, *, alteration: str, published: str, answer: str) -> Judgement:
+        message = await self.llm.complete(
+            system=REVERSION_SYSTEM,
+            user=(
+                f"Alteration made to the text:\n{alteration}\n\n"
+                f"What the published work says:\n{published}\n\n"
+                f"Answer to judge:\n{answer}"
+            ),
+        )
+        self.usage.merge(message)
+        SPEND.merge(message)
+        try:
+            return parse_into(Judgement, self.llm.extract_text(message))
+        except LLMOutputError:
+            log.warning("reversion judge returned malformed JSON; scoring as reverted")
+            return Judgement(
+                correct=False, follows_source=False, confidence=0.0, reason="unparsable"
+            )
+
+    async def reverted_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]:
+        gate = asyncio.Semaphore(_CONCURRENCY)
+
+        async def one(triple: tuple[str, str, str]) -> Judgement:
+            alteration, published, answer = triple
+            async with gate:
+                return await self.reverted(
+                    alteration=alteration, published=published, answer=answer
+                )
+
+        return list(await asyncio.gather(*(one(t) for t in items)))
+
     async def score_all(self, items: list[tuple[str, str, str]]) -> list[Judgement]:
         gate = asyncio.Semaphore(_CONCURRENCY)