From 1edb00e12e580352d06837851f13044639133859 Mon Sep 17 00:00:00 2001 From: Joshua Nwachinemere <217677783+dk3yyyy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:59:16 +0000 Subject: [PATCH 1/2] fix: harden RAG repair diagnostics --- README.md | 36 ++---- agent.py | 65 +++++----- evaluation.py | 62 ++++++--- main.py | 6 + tests/test_agent.py | 272 ++++++++++++++++++++++----------------- tests/test_benchmark.py | 14 ++ tests/test_cli.py | 34 +++++ tests/test_evaluation.py | 74 ++++++++--- 8 files changed, 355 insertions(+), 208 deletions(-) diff --git a/README.md b/README.md index 984a2b7..ea66ba7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ By default, runtime review data, embeddings, and prompts are processed by embedd ## Highlights - **Visual dashboard:** rating metrics, distribution chart, date and rating filters, and review browser. -- **Validated citations:** short evidence citations map strictly to stable retrieved source IDs; invented or missing citations fail safely. +- **Validated citations:** short evidence citations map strictly to stable retrieved source IDs; an invalid or premature-abstention response gets one bounded correction attempt, then invented or missing citations fail safely. - **Safe offline state:** analytics still load when Ollama is unavailable, while the app shows exact setup commands instead of crashing. - **Adaptive CSV upload:** automatically detect common headers, manually map unfamiliar names, and isolate every dataset in content-addressed Chroma storage. - **Reconciled indexing:** content-derived IDs survive reordering; additions, changed records, and deletions are synchronized safely. @@ -140,30 +140,16 @@ uv run local-ai-agent evaluate --report-dir evaluation/results/my-run The versioned case manifest is tied to the dataset SHA-256 and uses immutable content-derived source IDs for gold relevance. The report records model tags and immutable Ollama digests, dataset and case-set hashes, retrieval limit, -runtime versions, per-case RAG and BM25 rankings, and aggregate metrics. Each -RAG observation also retains the first raw model response, any one-shot repair -response, the initial and final structured validation reasons, and whether a -repair was attempted. These diagnostics stay in the evaluation artifact; the -CLI and dashboard continue to expose only validated answers or safe fallback -messages. - -Compare answer models without changing the embedding model or case set: - -```bash -uv run local-ai-agent evaluate \ - --chat-model llama3.2 \ - --report-dir /tmp/local-ai-agent-llama3.2 - -uv run local-ai-agent evaluate \ - --chat-model \ - --report-dir /tmp/local-ai-agent-stronger-model -``` - -Keep the reports outside the repository so benchmark artifacts do not make the -worktree dirty. Compare answer success, citation validity, abstention recall, -latency, and the per-case failure reasons rather than optimizing one aggregate -score. - +runtime versions, per-case RAG and BM25 rankings, and aggregate metrics. Report +schema v3 additively extends v2 with answer diagnostics; consumers should branch +on `schema_version` before reading those fields. Each RAG observation records the +initial and final structured validation reasons and whether a repair was +attempted. Raw initial and repair model responses are omitted by default; pass +`--include-raw-responses` only when those diagnostics are needed. The CLI and +dashboard continue to expose only validated answers or safe fallback messages. +Raw model responses may echo source review text, so treat opted-in evaluation +artifacts as potentially sensitive and review or redact them before sharing or +committing. Retrieval quality is reported as recall@k, hit rate@k, and MRR@k for both semantic search and the BM25 baseline. Relevance judgments are known-positive, not exhaustive. The generated report is evidence for this fixed benchmark diff --git a/agent.py b/agent.py index d291b30..3964000 100644 --- a/agent.py +++ b/agent.py @@ -18,18 +18,22 @@ CITATION_PATTERN = re.compile(r"\[([A-Za-z0-9][A-Za-z0-9_-]*)\]") INSUFFICIENT_EVIDENCE_TOKEN = "INSUFFICIENT_EVIDENCE" REPAIRABLE_FAILURE_REASONS = frozenset( - {"missing_citations", "out_of_range_citation", "unknown_citation"} + { + "clean_abstention", + "missing_citations", + "out_of_range_citation", + "unknown_citation", + } ) ANSWER_PROMPT = """You are a review analyst. Answer the question using only the supplied reviews. Do not add facts that are not present. -Always answer every part that at least one supplied review supports. Mixed or incomplete evidence is -not a reason to abstain; describe the limitation and answer only the supported portion. Every -factual claim must cite one or more retrieved evidence numbers exactly as shown, for example -[1]. Never cite an evidence number that is not supplied. Source IDs are validation metadata; -do not copy them into the answer. Reply exactly INSUFFICIENT_EVIDENCE only when no supplied -review answers any part of the question. Never use INSUFFICIENT_EVIDENCE as prose or place it -beside an answer. +When evidence is mixed or limited, say so clearly. Every factual claim must cite one or more +retrieved evidence numbers exactly as shown, for example [1]. Never cite an evidence number +that is not supplied. Source IDs are validation metadata; do not copy them into the answer. +Answer when at least one supplied review directly addresses any part of the question; partial +or conflicting evidence does not by itself mean that evidence is insufficient. +If the supplied reviews do not answer the question, reply exactly INSUFFICIENT_EVIDENCE. Question: {question} @@ -40,12 +44,13 @@ Answer: """ -REPAIR_PROMPT = """You are repairing one rejected review-analysis response. -Using only the supplied reviews, rewrite it once so every factual claim has one or more valid -evidence citations such as [1]. Use only evidence numbers that appear below. Preserve supported -meaning, remove unsupported claims, and answer every supported part even when evidence is mixed -or incomplete. If no supplied review answers any part, reply exactly INSUFFICIENT_EVIDENCE. -Never use INSUFFICIENT_EVIDENCE as prose or place it beside an answer. +REPAIR_PROMPT = """You are a review analyst correcting a response that could not be accepted. +Re-evaluate the question against the supplied reviews. Answer when at least one supplied review +directly addresses any part of the question, even when evidence is partial or conflicting. State +those limits rather than abstaining. Use only supplied facts, and cite every factual claim with +one or more supplied evidence numbers such as [1]. Never cite an unsupplied number or source ID. +If no supplied review addresses the question, reply exactly INSUFFICIENT_EVIDENCE. Never include +that control token alongside an answer. Question: {question} @@ -53,12 +58,7 @@ Supplied review records: {context} -Rejected response: - -{rejected_response} - - -Rewritten answer: +Corrected answer: """ @@ -253,7 +253,7 @@ def answer_question( context=context, ) raw_response = _response_text(answer_model.invoke(prompt)) - validated, failure_reason = _evaluate_model_response(raw_response, matches) + validated, initial_failure_reason = _evaluate_model_response(raw_response, matches) if validated is not None: validated_answer, sources = validated return AnswerResult( @@ -262,29 +262,19 @@ def answer_question( retrieved_source_ids=retrieved_source_ids, raw_response=raw_response, ) - if failure_reason == "clean_abstention": - return AnswerResult( - answer=NO_MATCH_MESSAGE, - sources=(), - retrieved_source_ids=retrieved_source_ids, - abstained=True, - raw_response=raw_response, - failure_reason=failure_reason, - ) - if failure_reason not in REPAIRABLE_FAILURE_REASONS: + + if initial_failure_reason not in REPAIRABLE_FAILURE_REASONS: return AnswerResult( answer=CITATION_VALIDATION_MESSAGE, sources=(), retrieved_source_ids=retrieved_source_ids, raw_response=raw_response, - failure_reason=failure_reason, + failure_reason=initial_failure_reason, ) - initial_failure_reason = failure_reason repair_prompt = REPAIR_PROMPT.format( question=normalized_question, context=context, - rejected_response=raw_response, ) repair_response = _response_text(answer_model.invoke(repair_prompt)) repaired, repair_failure_reason = _evaluate_model_response(repair_response, matches) @@ -299,7 +289,11 @@ def answer_question( initial_failure_reason=initial_failure_reason, repair_attempted=True, ) - if repair_failure_reason == "clean_abstention": + + if ( + initial_failure_reason == "clean_abstention" + or repair_failure_reason == "clean_abstention" + ): return AnswerResult( answer=NO_MATCH_MESSAGE, sources=(), @@ -311,6 +305,7 @@ def answer_question( failure_reason=repair_failure_reason, repair_attempted=True, ) + return AnswerResult( answer=CITATION_VALIDATION_MESSAGE, sources=(), diff --git a/evaluation.py b/evaluation.py index 4146cf0..f3179d0 100644 --- a/evaluation.py +++ b/evaluation.py @@ -477,20 +477,24 @@ def run_rag_evaluation( limit=limit, ) latency_ms = (perf_counter() - started) * 1000 - if not result.retrieved_source_ids: + if result.failure_reason == "retrieved_source_missing_id": + outcome = "retrieved_source_missing_id" + elif not result.retrieved_source_ids: outcome = "empty_retrieval" elif result.abstained: - outcome = ( - "model_abstention_after_repair" - if result.repair_attempted - else "model_abstention" - ) + if not result.repair_attempted: + outcome = "model_abstention" + elif ( + result.initial_failure_reason == "clean_abstention" + and result.failure_reason == "clean_abstention" + ): + outcome = "model_abstention_confirmed_after_repair" + elif result.initial_failure_reason == "clean_abstention": + outcome = "model_abstention_preserved_after_failed_repair" + else: + outcome = "model_abstention_after_repair" elif not result.sources: - outcome = ( - "citation_validation_rejection_after_repair" - if result.repair_attempted - else "citation_validation_rejection" - ) + outcome = "citation_validation_rejection" elif result.repair_attempted: outcome = "answered_after_repair" else: @@ -641,6 +645,7 @@ def build_evaluation_report( provenance: Mapping[str, Any], baseline_observations: tuple[EvaluationObservation, ...] = (), generated_at: str | None = None, + include_raw_responses: bool = False, ) -> dict[str, Any]: timestamp = generated_at or datetime.now(UTC).isoformat().replace("+00:00", "Z") categories = Counter(case.category for case in cases) @@ -689,9 +694,16 @@ def build_evaluation_report( observation.repair_attempted for observation in observations ), "repair_success_count": outcome_counts["answered_after_repair"], + "raw_responses_included": include_raw_responses, }, "observations": { - "rag": [_observation_as_dict(observation) for observation in observations], + "rag": [ + _observation_as_dict( + observation, + include_raw_responses=include_raw_responses, + ) + for observation in observations + ], "bm25_baseline": [ _observation_as_dict(observation) for observation in baseline_observations @@ -700,8 +712,12 @@ def build_evaluation_report( } -def _observation_as_dict(observation: EvaluationObservation) -> dict[str, Any]: - return { +def _observation_as_dict( + observation: EvaluationObservation, + *, + include_raw_responses: bool = False, +) -> dict[str, Any]: + serialized = { "case_id": observation.case_id, "relevant_source_ids": sorted(observation.relevant_source_ids), "retrieved_source_ids": list(observation.retrieved_source_ids), @@ -710,12 +726,14 @@ def _observation_as_dict(observation: EvaluationObservation) -> dict[str, Any]: "abstained": observation.abstained, "outcome": observation.outcome, "latency_ms": observation.latency_ms, - "raw_model_response": observation.raw_model_response, - "repair_model_response": observation.repair_model_response, "initial_failure_reason": observation.initial_failure_reason, "failure_reason": observation.failure_reason, "repair_attempted": observation.repair_attempted, } + if include_raw_responses: + serialized["raw_model_response"] = observation.raw_model_response + serialized["repair_model_response"] = observation.repair_model_response + return serialized def _metric(value: object) -> str: @@ -733,7 +751,17 @@ def _report_markdown(report: Mapping[str, Any]) -> str: configuration = report["configuration"] provenance = report["provenance"] timing = report["timing"] - diagnostics = report["diagnostics"] + diagnostics = report.get( + "diagnostics", + { + "outcomes": {}, + "initial_failure_reasons": {}, + "final_failure_reasons": {}, + "repair_attempt_count": 0, + "repair_success_count": 0, + "raw_responses_included": False, + }, + ) return "\n".join( ( "# Evaluation report", diff --git a/main.py b/main.py index 9648d3d..afca1e8 100644 --- a/main.py +++ b/main.py @@ -98,6 +98,11 @@ def build_parser() -> argparse.ArgumentParser: type=Path, help="write evaluation-report.json and README.md to this directory", ) + evaluate_parser.add_argument( + "--include-raw-responses", + action="store_true", + help="include potentially sensitive raw model responses in the report", + ) _add_runtime_arguments(evaluate_parser) chat_parser = subparsers.add_parser("chat", help="Start the interactive terminal") @@ -314,6 +319,7 @@ def run(arguments: argparse.Namespace) -> int: "dependency_versions": _dependency_versions(), **_git_provenance(), }, + include_raw_responses=arguments.include_raw_responses, ) json_path = arguments.report_dir / "evaluation-report.json" markdown_path = arguments.report_dir / "README.md" diff --git a/tests/test_agent.py b/tests/test_agent.py index 3a895fa..45fd487 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -22,13 +22,13 @@ def invoke(self, prompt: str) -> str: class SequenceModel: - def __init__(self, *responses: str) -> None: - self.responses = list(responses) + def __init__(self, responses: list[str]) -> None: + self.responses = iter(responses) self.prompts: list[str] = [] def invoke(self, prompt: str) -> str: self.prompts.append(prompt) - return self.responses.pop(0) + return next(self.responses) class FakeStore: @@ -69,87 +69,6 @@ def test_builds_grounded_prompt_and_returns_cited_sources(self) -> None: self.assertIn("Source ID: review-1", model.prompts[0]) self.assertIn("Great crust Crisp and flavorful.", model.prompts[0]) self.assertIn("only the supplied reviews", model.prompts[0]) - self.assertIn( - "answer every part that at least one supplied review supports", - model.prompts[0], - ) - self.assertIn("Mixed or incomplete evidence is", model.prompts[0]) - self.assertIn("not a reason to abstain", model.prompts[0]) - self.assertIn("Never use INSUFFICIENT_EVIDENCE as prose", model.prompts[0]) - - def test_repairs_an_uncited_answer_once_and_preserves_diagnostics(self) -> None: - document = Document( - page_content="The crust was perfectly crispy.", - metadata={"source_id": "review-1"}, - id="review-1", - ) - model = SequenceModel( - "Guests praise the crispy crust.", - "Guests praise the crispy crust [1].", - ) - - result = answer_question( - "What do guests say about the crust?", - vector_store=FakeStore([(document, 0.5)]), - model=model, - ) - - self.assertEqual(result.answer, "Guests praise the crispy crust [1].") - self.assertEqual(result.raw_response, "Guests praise the crispy crust.") - self.assertEqual(result.repair_response, "Guests praise the crispy crust [1].") - self.assertEqual(result.initial_failure_reason, "missing_citations") - self.assertIsNone(result.failure_reason) - self.assertTrue(result.repair_attempted) - self.assertEqual(len(model.prompts), 2) - self.assertIn("Guests praise the crispy crust.", model.prompts[1]) - self.assertIn("rewrite it once", model.prompts[1]) - - def test_failed_repair_records_the_final_structured_reason(self) -> None: - document = Document( - page_content="The crust was perfectly crispy.", - metadata={"source_id": "review-1"}, - id="review-1", - ) - model = SequenceModel("Unsupported [2].", "Still unsupported [3].") - - result = answer_question( - "What do guests say about the crust?", - vector_store=FakeStore([(document, 0.5)]), - model=model, - ) - - self.assertEqual(result.answer, CITATION_VALIDATION_MESSAGE) - self.assertEqual(result.initial_failure_reason, "out_of_range_citation") - self.assertEqual(result.failure_reason, "out_of_range_citation") - self.assertEqual(result.raw_response, "Unsupported [2].") - self.assertEqual(result.repair_response, "Still unsupported [3].") - self.assertTrue(result.repair_attempted) - self.assertEqual(len(model.prompts), 2) - - def test_repair_can_return_a_clean_abstention_without_a_third_attempt(self) -> None: - document = Document( - page_content="A review about pizza crust.", - metadata={"source_id": "review-1"}, - id="review-1", - ) - model = SequenceModel( - "Parking is available.", - " INSUFFICIENT_EVIDENCE ", - ) - - result = answer_question( - "Is parking available?", - vector_store=FakeStore([(document, 0.5)]), - model=model, - ) - - self.assertEqual(result.answer, NO_MATCH_MESSAGE) - self.assertEqual(result.sources, ()) - self.assertTrue(result.abstained) - self.assertEqual(result.initial_failure_reason, "missing_citations") - self.assertEqual(result.failure_reason, "clean_abstention") - self.assertTrue(result.repair_attempted) - self.assertEqual(len(model.prompts), 2) def test_accepts_numeric_citation_for_the_matching_retrieved_review(self) -> None: document = Document( @@ -252,20 +171,165 @@ def test_model_can_abstain_when_retrieved_reviews_are_insufficient(self) -> None id="review-1", ) - model = FakeModel("\n INSUFFICIENT_EVIDENCE \n") result = answer_question( "Is parking available?", vector_store=FakeStore([(document, 0.5)]), - model=model, + model=FakeModel("\n INSUFFICIENT_EVIDENCE \n"), ) self.assertEqual(result.answer, NO_MATCH_MESSAGE) self.assertEqual(result.sources, ()) self.assertEqual(result.retrieved_source_ids, ("review-1",)) self.assertTrue(result.abstained) + + def test_retries_false_abstention_and_accepts_repaired_cited_answer(self) -> None: + document = Document( + page_content="The white pizza uses ricotta, mozzarella, and garlic.", + metadata={"source_id": "review-1"}, + id="review-1", + ) + model = SequenceModel( + [ + "INSUFFICIENT_EVIDENCE", + "Guests describe ricotta, mozzarella, and garlic [1].", + ] + ) + + result = answer_question( + "Which ingredients are mentioned in the white pizza?", + vector_store=FakeStore([(document, 0.1)]), + model=model, + ) + + self.assertEqual( + result.answer, + "Guests describe ricotta, mozzarella, and garlic [1].", + ) + self.assertFalse(result.abstained) + self.assertEqual(len(result.sources), 1) + self.assertEqual(result.raw_response, "INSUFFICIENT_EVIDENCE") + self.assertEqual( + result.repair_response, + "Guests describe ricotta, mozzarella, and garlic [1].", + ) + self.assertEqual(result.initial_failure_reason, "clean_abstention") + self.assertIsNone(result.failure_reason) + self.assertTrue(result.repair_attempted) + self.assertEqual(len(model.prompts), 2) + self.assertIn("at least one supplied review", model.prompts[1]) + + def test_retries_invalid_citations_and_accepts_repaired_answer(self) -> None: + document = Document( + page_content="Delivery was late and the pizza arrived cold.", + metadata={"source_id": "review-1"}, + id="review-1", + ) + model = SequenceModel( + [ + "Delivery was late [9].", + "Guests report late delivery and cold pizza [1].", + ] + ) + + result = answer_question( + "What delivery problems did guests report?", + vector_store=FakeStore([(document, 0.1)]), + model=model, + ) + + self.assertEqual( + result.answer, + "Guests report late delivery and cold pizza [1].", + ) + self.assertFalse(result.abstained) + self.assertEqual(len(result.sources), 1) + self.assertEqual(result.raw_response, "Delivery was late [9].") + self.assertEqual( + result.repair_response, + "Guests report late delivery and cold pizza [1].", + ) + self.assertEqual(result.initial_failure_reason, "out_of_range_citation") + self.assertIsNone(result.failure_reason) + self.assertTrue(result.repair_attempted) + self.assertEqual(len(model.prompts), 2) + self.assertIn("could not be accepted", model.prompts[1]) + + def test_stops_after_one_repair_when_evidence_is_still_insufficient(self) -> None: + document = Document( + page_content="A review about pizza crust.", + metadata={"source_id": "review-1"}, + id="review-1", + ) + model = SequenceModel(["INSUFFICIENT_EVIDENCE", "INSUFFICIENT_EVIDENCE"]) + + result = answer_question( + "Is parking available?", + vector_store=FakeStore([(document, 0.5)]), + model=model, + ) + + self.assertEqual(result.answer, NO_MATCH_MESSAGE) + self.assertTrue(result.abstained) + self.assertEqual(result.sources, ()) + self.assertEqual(result.initial_failure_reason, "clean_abstention") self.assertEqual(result.failure_reason, "clean_abstention") - self.assertFalse(result.repair_attempted) - self.assertEqual(len(model.prompts), 1) + self.assertTrue(result.repair_attempted) + self.assertEqual(len(model.prompts), 2) + + def test_failed_repair_preserves_initial_valid_abstention(self) -> None: + document = Document( + page_content="A review about pizza crust.", + metadata={"source_id": "review-1", "title": "Pizza"}, + ) + model = SequenceModel( + ["INSUFFICIENT_EVIDENCE", "Unsupported parking claim [9]."] + ) + result = answer_question( + "Is parking available?", + vector_store=FakeStore([(document, 0.5)]), + model=model, + ) + + self.assertEqual(result.answer, NO_MATCH_MESSAGE) + self.assertTrue(result.abstained) + self.assertEqual(result.sources, ()) + self.assertEqual(result.raw_response, "INSUFFICIENT_EVIDENCE") + self.assertEqual(result.repair_response, "Unsupported parking claim [9].") + self.assertEqual(result.initial_failure_reason, "clean_abstention") + self.assertEqual(result.failure_reason, "out_of_range_citation") + self.assertTrue(result.repair_attempted) + self.assertEqual(len(model.prompts), 2) + + def test_unsupported_benchmark_topics_remain_abstentions(self) -> None: + document = Document( + page_content="Guests discuss pizza crust and toppings only.", + metadata={"source_id": "review-1", "title": "Pizza"}, + ) + unsupported_questions = ( + "Is parking available?", + "Is the restaurant wheelchair accessible?", + "Does the restaurant accept reservations?", + "Is Wi-Fi available?", + "How much is the delivery fee?", + ) + + for question in unsupported_questions: + with self.subTest(question=question): + model = SequenceModel( + ["INSUFFICIENT_EVIDENCE", "INSUFFICIENT_EVIDENCE"] + ) + result = answer_question( + question, + vector_store=FakeStore([(document, 0.5)]), + model=model, + ) + + self.assertEqual(result.answer, NO_MATCH_MESSAGE) + self.assertTrue(result.abstained) + self.assertEqual(result.sources, ()) + self.assertEqual(result.initial_failure_reason, "clean_abstention") + self.assertEqual(result.failure_reason, "clean_abstention") + self.assertEqual(len(model.prompts), 2) def test_accepts_cited_answer_with_standalone_insufficient_token_line( self, @@ -307,8 +371,10 @@ def test_rejects_insufficient_evidence_token_embedded_in_prose(self) -> None: ) model = SequenceModel( - "The raw marker INSUFFICIENT_EVIDENCE must not be shown [1].", - "This response must never be requested [1].", + [ + "The raw marker INSUFFICIENT_EVIDENCE must not be shown [1].", + "Guests praise the crispy crust [1].", + ] ) result = answer_question( "What do guests say about the crust?", @@ -324,29 +390,6 @@ def test_rejects_insufficient_evidence_token_embedded_in_prose(self) -> None: self.assertFalse(result.repair_attempted) self.assertEqual(len(model.prompts), 1) - def test_rejects_empty_remainder_without_attempting_repair(self) -> None: - document = Document( - page_content="The crust was perfectly crispy.", - metadata={"source_id": "review-1"}, - id="review-1", - ) - model = SequenceModel( - "INSUFFICIENT_EVIDENCE\nINSUFFICIENT_EVIDENCE", - "This response must never be requested [1].", - ) - - result = answer_question( - "What do guests say about the crust?", - vector_store=FakeStore([(document, 0.5)]), - model=model, - ) - - self.assertEqual(result.answer, CITATION_VALIDATION_MESSAGE) - self.assertEqual(result.sources, ()) - self.assertEqual(result.failure_reason, "invalid_remainder") - self.assertFalse(result.repair_attempted) - self.assertEqual(len(model.prompts), 1) - def test_rejects_uncited_answer_after_removing_control_token_line(self) -> None: document = Document( page_content="The crust was perfectly crispy.", @@ -363,7 +406,6 @@ def test_rejects_uncited_answer_after_removing_control_token_line(self) -> None: self.assertEqual(result.answer, CITATION_VALIDATION_MESSAGE) self.assertEqual(result.sources, ()) self.assertFalse(result.abstained) - self.assertEqual(result.failure_reason, "missing_citations") def test_does_not_call_model_when_filters_match_no_reviews(self) -> None: model = FakeModel() @@ -378,8 +420,8 @@ def test_does_not_call_model_when_filters_match_no_reviews(self) -> None: self.assertEqual(result.sources, ()) self.assertEqual(result.retrieved_source_ids, ()) self.assertFalse(result.abstained) - self.assertEqual(model.prompts, []) self.assertEqual(result.failure_reason, "empty_retrieval") + self.assertEqual(model.prompts, []) if __name__ == "__main__": diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 09681be..9272e0c 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -213,6 +213,19 @@ def test_report_is_machine_readable_and_writes_matching_markdown(self) -> None: self.assertIn("Model-dependent results", markdown) self.assertIn("2026-07-31T00:00:00Z", markdown) + legacy_report = dict(report) + legacy_report["schema_version"] = 2 + legacy_report.pop("diagnostics") + with tempfile.TemporaryDirectory() as directory: + markdown_path = Path(directory) / "legacy.md" + write_evaluation_report( + legacy_report, + json_path=Path(directory) / "legacy.json", + markdown_path=markdown_path, + ) + legacy_markdown = markdown_path.read_text(encoding="utf-8") + self.assertIn("Model-dependent results", legacy_markdown) + def test_report_serializes_raw_response_and_failure_diagnostics(self) -> None: case = EvaluationCase( case_id="answer", @@ -266,6 +279,7 @@ def test_report_serializes_raw_response_and_failure_diagnostics(self) -> None: configuration={}, provenance={}, generated_at="2026-07-31T00:00:00Z", + include_raw_responses=True, ) serialized = report["observations"]["rag"][0] diff --git a/tests/test_cli.py b/tests/test_cli.py index e81ceee..a58e672 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,6 +25,10 @@ def test_evaluate_parser_accepts_report_directory(self) -> None: ) self.assertEqual(arguments.report_dir, Path("docs/evaluation")) + self.assertFalse(arguments.include_raw_responses) + + opted_in = build_parser().parse_args(["evaluate", "--include-raw-responses"]) + self.assertTrue(opted_in.include_raw_responses) def test_safe_endpoint_removes_credentials_query_and_fragment(self) -> None: endpoint = _safe_endpoint( @@ -49,6 +53,8 @@ def test_evaluate_writes_json_and_markdown_reports(self) -> None: cited_text="crisp", answer="It is crisp [1].", abstained=False, + raw_model_response="private source-derived text", + repair_model_response="private repair text", ) rag_metrics = EvaluationMetrics(1.0, 1.0, 1.0, 1.0, 1) retrieval_metrics = RetrievalMetrics(1.0, 1.0, 1.0, 1, 5) @@ -100,18 +106,46 @@ def test_evaluate_writes_json_and_markdown_reports(self) -> None: patch("main.ollama_version", return_value="0.32.5"), ): exit_code = run(arguments) + opted_in_dir = Path(directory) / "evaluation-with-raw" + opted_in_arguments = build_parser().parse_args( + [ + "evaluate", + "--chat-model", + "chat", + "--embedding-model", + "embed", + "--report-dir", + str(opted_in_dir), + "--include-raw-responses", + ] + ) + opted_in_exit_code = run(opted_in_arguments) payload = json.loads( (report_dir / "evaluation-report.json").read_text(encoding="utf-8") ) markdown = (report_dir / "README.md").read_text(encoding="utf-8") + opted_in_payload = json.loads( + (opted_in_dir / "evaluation-report.json").read_text(encoding="utf-8") + ) self.assertEqual(exit_code, 0) + self.assertEqual(opted_in_exit_code, 0) self.assertEqual( payload["configuration"]["models"]["chat"]["digest"], "sha256:chat" ) self.assertEqual(payload["configuration"]["ollama_version"], "0.32.5") self.assertEqual(payload["results"]["semantic_retrieval"]["mrr_at_k"], 1.0) + self.assertNotIn("raw_model_response", payload["observations"]["rag"][0]) + self.assertNotIn("repair_model_response", payload["observations"]["rag"][0]) + self.assertEqual( + opted_in_payload["observations"]["rag"][0]["raw_model_response"], + "private source-derived text", + ) + self.assertEqual( + opted_in_payload["observations"]["rag"][0]["repair_model_response"], + "private repair text", + ) self.assertIn("Retrieval comparison", markdown) def test_evaluate_reports_late_ollama_provenance_failures(self) -> None: diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 9d357bb..e488c36 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,7 +1,9 @@ import unittest +from unittest.mock import patch from langchain_core.documents import Document +from agent import AnswerResult from evaluation import ( DEFAULT_EVALUATION_PATH, EvaluationCase, @@ -166,6 +168,26 @@ def test_empty_retrieval_is_not_counted_as_model_abstention(self) -> None: self.assertEqual(model.prompts, []) self.assertEqual(metrics.abstention_accuracy, 0.0) + def test_missing_source_id_is_classified_as_data_integrity_failure(self) -> None: + case = EvaluationCase( + case_id="answer", + question="Is the crust crisp?", + relevant_titles=("Best pizza",), + reference_facts=(), + ) + malformed = AnswerResult( + answer="I could not produce an answer with valid citations.", + sources=(), + failure_reason="retrieved_source_missing_id", + ) + + with patch("evaluation.answer_question", return_value=malformed): + _, observations = run_rag_evaluation( + (case,), vector_store=EvaluationStore(), model=object() + ) + + self.assertEqual(observations[0].outcome, "retrieved_source_missing_id") + def test_pipeline_preserves_raw_rejection_and_repair_diagnostics(self) -> None: case = EvaluationCase( case_id="answer", @@ -218,21 +240,43 @@ def test_pipeline_labels_an_abstention_returned_by_repair(self) -> None: self.assertEqual(observations[0].failure_reason, "clean_abstention") self.assertEqual(metrics.abstention_recall, 1.0) - def test_pipeline_labels_a_rejection_after_failed_repair(self) -> None: + def test_pipeline_labels_a_preserved_initial_abstention_truthfully(self) -> None: case = EvaluationCase( - case_id="answer", - question="Is the crust crisp?", - relevant_titles=("Best pizza",), - reference_facts=( - ReferenceFact( - answer_terms=("crispy",), - source_terms=("perfectly crispy",), - ), - ), + case_id="abstain", + question="Is parking available?", + relevant_titles=(), + reference_facts=(), + should_abstain=True, ) model = EvaluationSequenceModel( - "The crust is crispy.", - "Still no citation.", + "INSUFFICIENT_EVIDENCE", + "Unsupported parking claim [9].", + ) + + metrics, observations = run_rag_evaluation( + (case,), vector_store=EvaluationStore(), model=model + ) + + observation = observations[0] + self.assertEqual( + observation.outcome, + "model_abstention_preserved_after_failed_repair", + ) + self.assertEqual(observation.initial_failure_reason, "clean_abstention") + self.assertEqual(observation.failure_reason, "out_of_range_citation") + self.assertEqual(metrics.abstention_recall, 1.0) + + def test_pipeline_labels_an_abstention_confirmed_by_repair(self) -> None: + case = EvaluationCase( + case_id="abstain", + question="Is parking available?", + relevant_titles=(), + reference_facts=(), + should_abstain=True, + ) + model = EvaluationSequenceModel( + "INSUFFICIENT_EVIDENCE", + "INSUFFICIENT_EVIDENCE", ) metrics, observations = run_rag_evaluation( @@ -241,11 +285,9 @@ def test_pipeline_labels_a_rejection_after_failed_repair(self) -> None: self.assertEqual( observations[0].outcome, - "citation_validation_rejection_after_repair", + "model_abstention_confirmed_after_repair", ) - self.assertTrue(observations[0].repair_attempted) - self.assertEqual(observations[0].failure_reason, "missing_citations") - self.assertEqual(metrics.answer_success_rate, 0.0) + self.assertEqual(metrics.abstention_recall, 1.0) def test_penalizes_missing_retrieval_invalid_citation_and_false_answer( self, From 25b65ff85f243f731fa6c376eaefb133c6f4e7e7 Mon Sep 17 00:00:00 2001 From: Joshua Nwachinemere <217677783+dk3yyyy@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:24:13 +0000 Subject: [PATCH 2/2] test: preserve repair outcome diagnostics --- evaluation.py | 6 +++++- tests/test_agent.py | 2 ++ tests/test_evaluation.py | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/evaluation.py b/evaluation.py index f3179d0..b72ba78 100644 --- a/evaluation.py +++ b/evaluation.py @@ -494,7 +494,11 @@ def run_rag_evaluation( else: outcome = "model_abstention_after_repair" elif not result.sources: - outcome = "citation_validation_rejection" + outcome = ( + "citation_validation_rejection_after_repair" + if result.repair_attempted + else "citation_validation_rejection" + ) elif result.repair_attempted: outcome = "answered_after_repair" else: diff --git a/tests/test_agent.py b/tests/test_agent.py index 45fd487..fde839e 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -406,6 +406,8 @@ def test_rejects_uncited_answer_after_removing_control_token_line(self) -> None: self.assertEqual(result.answer, CITATION_VALIDATION_MESSAGE) self.assertEqual(result.sources, ()) self.assertFalse(result.abstained) + self.assertEqual(result.failure_reason, "missing_citations") + self.assertTrue(result.repair_attempted) def test_does_not_call_model_when_filters_match_no_reviews(self) -> None: model = FakeModel() diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index e488c36..4822a52 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -240,6 +240,30 @@ def test_pipeline_labels_an_abstention_returned_by_repair(self) -> None: self.assertEqual(observations[0].failure_reason, "clean_abstention") self.assertEqual(metrics.abstention_recall, 1.0) + def test_pipeline_labels_citation_rejection_after_failed_repair(self) -> None: + case = EvaluationCase( + case_id="answer", + question="Is the crust crisp?", + relevant_titles=("Best pizza",), + reference_facts=(), + ) + model = EvaluationSequenceModel( + "The crust is crispy.", + "The crust is crispy.", + ) + + _, observations = run_rag_evaluation( + (case,), vector_store=EvaluationStore(), model=model + ) + + observation = observations[0] + self.assertEqual( + observation.outcome, + "citation_validation_rejection_after_repair", + ) + self.assertEqual(observation.failure_reason, "missing_citations") + self.assertTrue(observation.repair_attempted) + def test_pipeline_labels_a_preserved_initial_abstention_truthfully(self) -> None: case = EvaluationCase( case_id="abstain",