fix(core): address release review findings for v0.4.0

scripts/eval_e2e.py

@@ -115,8 +115,14 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None:
         return
 
     valid = [r for r in results if r.error is None]
-    answered = sum(1 for r in valid if r.answer is not None)
+    answered = sum(
+        1 for r in valid if r.answer is not None and not r.no_relevant_context
+    )
+    answered_no_ctx = sum(
+        1 for r in valid if r.answer is not None and r.no_relevant_context
+    )
     no_context = sum(1 for r in valid if r.no_relevant_context)
+    unanswered = sum(1 for r in valid if r.answer is None)
     avg_sources = sum(r.num_sources for r in valid) / evaluated
 
     durations = sorted(r.duration_ms for r in valid)
@@ -129,8 +135,10 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None:
     print(f"Questions:         {total} ({errors} errors)")
     print(f"Duration:          {elapsed_s:.1f}s total")
     print("")
-    print(f"Answered:          {answered}/{evaluated} ({answered / evaluated:.0%})")
-    print(f"No context:        {no_context}/{evaluated}")
+    print(f"Answered (grounded): {answered}/{evaluated} ({answered / evaluated:.0%})")
+    print(f"Answered (no ctx):   {answered_no_ctx}/{evaluated} (LLM without retrieval)")
+    print(f"Unanswered:          {unanswered}/{evaluated}")
+    print(f"No context:          {no_context}/{evaluated}")
     print(f"Avg sources:       {avg_sources:.1f}")
     print(f"Latency p50:       {p50:.0f}ms")
     print(f"Latency p95:       {p95:.0f}ms")
@@ -141,9 +149,13 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None:
         print("\nBy category:")
         for cat in categories:
             cat_results = [r for r in valid if r.category == cat]
-            cat_answered = sum(1 for r in cat_results if r.answer is not None)
+            cat_grounded = sum(
+                1
+                for r in cat_results
+                if r.answer is not None and not r.no_relevant_context
+            )
             print(
-                f"  {cat:<15} answered={cat_answered}/{len(cat_results)}  n={len(cat_results)}"
+                f"  {cat:<15} grounded={cat_grounded}/{len(cat_results)}  n={len(cat_results)}"
             )
 
     print(f"{'=' * 60}")

scripts/eval_retrieval.py

@@ -2,7 +2,9 @@
 """Retrieval-only evaluation harness (TECH-002).
 
 Reads a JSONL dataset, calls the Vektra search API for each question,
-and computes retrieval quality metrics. No LLM calls are made.
+and computes retrieval quality metrics.
+By default uses /api/v1/search (no LLM). With --use-pipeline it calls
+/api/v1/query which runs the full RAG pipeline including LLM.
 
 Usage:
     python scripts/eval_retrieval.py [OPTIONS]
@@ -17,6 +19,7 @@
 import os
 import sys
 import time
+import unicodedata
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -56,22 +59,35 @@ def load_dataset(path: str) -> list[dict]:
     return entries
 
 
+def _normalize(s: str) -> str:
+    """Lowercase, strip diacritics and apostrophes for keyword matching."""
+    nfkd = unicodedata.normalize("NFKD", s.casefold())
+    base = "".join(c for c in nfkd if not unicodedata.combining(c))
+    for ch in ("'", "\u2019", "\u2018", "\u02bc", "`"):
+        base = base.replace(ch, "")
+    return base
+
+
 def chunk_matches_keywords(text: str, keywords: list[str]) -> bool:
-    """Check if a chunk's text contains any of the expected keywords (case-insensitive)."""
-    text_lower = text.lower()
-    return any(kw.lower() in text_lower for kw in keywords)
+    """Check if a chunk's text contains any of the expected keywords.
+
+    Matching is case-insensitive and diacritic-insensitive so that
+    ASCII dataset entries like "liberta'" match Unicode text "libertà".
+    """
+    text_norm = _normalize(text)
+    return any(_normalize(kw) in text_norm for kw in keywords)
 
 
 def evaluate_question(
     client: httpx.Client,
     entry: dict,
     top_k: int,
     search_mode: str,
-    use_query: bool = False,
+    use_pipeline: bool = False,
 ) -> QuestionResult:
     """Run a single search/query and compute retrieval metrics.
 
-    When use_query=True, calls /api/v1/query (includes reranker) and
+    When use_pipeline=True, calls /api/v1/query (includes reranker) and
     evaluates on the returned sources. Otherwise calls /api/v1/search
     (raw vector search, no reranker).
     """
@@ -83,7 +99,7 @@ def evaluate_question(
     language = entry.get("language", "unknown")
 
     try:
-        if use_query:
+        if use_pipeline:
             resp = client.post(
                 "/api/v1/query",
                 json={
@@ -295,9 +311,9 @@ def main() -> None:
         help="Search mode (default: hybrid)",
     )
     parser.add_argument(
-        "--use-query",
+        "--use-pipeline",
         action="store_true",
-        help="Use /api/v1/query (with reranker) instead of /api/v1/search",
+        help="Use /api/v1/query (full RAG pipeline with reranker + LLM) instead of /api/v1/search (raw vector search). Results reflect the pipeline's source selection, not pure retrieval.",
     )
     args = parser.parse_args()
 
@@ -323,11 +339,13 @@ def main() -> None:
 
     print(f"Loaded {len(dataset)} questions from {args.dataset}")
     mode_info = (
-        "query (with reranker)" if args.use_query else f"search mode={args.search_mode}"
+        "pipeline (reranker + LLM)"
+        if args.use_pipeline
+        else f"search mode={args.search_mode}"
     )
     print(f"API: {api_url}  top_k={args.top_k}  {mode_info}")
 
-    timeout = 120.0 if args.use_query else 30.0
+    timeout = 120.0 if args.use_pipeline else 30.0
     client = httpx.Client(
         base_url=api_url,
         headers={"Authorization": f"Bearer {api_key}"},
@@ -338,7 +356,7 @@ def main() -> None:
     t0 = time.monotonic()
     for i, entry in enumerate(dataset):
         result = evaluate_question(
-            client, entry, args.top_k, args.search_mode, use_query=args.use_query
+            client, entry, args.top_k, args.search_mode, use_pipeline=args.use_pipeline
         )
         results.append(result)
         status = "HIT" if result.hit else ("ERR" if result.error else "MISS")

tests/eval/dataset.jsonl

@@ -46,7 +46,7 @@
 {"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"}
 {"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"}
 {"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"}
-{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"}
+{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": ["pena di morte", "non e' ammessa"], "namespace": "default", "category": "factual", "language": "it"}
 {"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"}
 {"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"}
 {"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"}

vektra-admin/src/vektra_admin/api.py

@@ -466,6 +466,7 @@ class ConversationTurnDetail(BaseModel):
 async def get_conversation_turns(
     conversation_id: UUID,
     request: Request,
+    background_tasks: BackgroundTasks,
     _key: ApiKeyInfo = Depends(require_scope("admin")),
 ) -> Any:
     """Return decrypted conversation turns with full metadata (admin only)."""
@@ -488,6 +489,24 @@ async def get_conversation_turns(
     if turns is None:
         raise HTTPException(status_code=404, detail="Conversation not found")
 
+    # Audit log: sensitive content access
+    request_id = getattr(request.state, "request_id", None)
+    if request_id:
+        background_tasks.add_task(
+            _audit.log_event,
+            key_id=_key.key_id,
+            endpoint=f"/api/v1/admin/conversations/{conversation_id}/turns",
+            method="GET",
+            status_code=200,
+            request_id=request_id,
+            action="conversation_turns_read",
+            log_metadata={
+                "namespace": getattr(request.state, "rls_namespace", None),
+                "conversation_id": str(conversation_id),
+                "turn_count": len(turns),
+            },
+        )
+
     return turns
 
 

vektra-admin/tests/test_admin_turns.py

@@ -38,14 +38,23 @@ async def test_returns_turns_from_persistent_store():
 
     request = MagicMock()
     request.app.state = app_state
+    request.state.request_id = "test-req-id"
 
+    background_tasks = MagicMock()
     key_info = MagicMock()
     key_info.scopes = ["admin"]
 
-    result = await get_conversation_turns(cid, request, key_info)
+    result = await get_conversation_turns(cid, request, background_tasks, key_info)
     assert result == turns
     conv_store.get_turns_detail.assert_called_once_with(cid)
 
+    # Verify audit log was queued
+    background_tasks.add_task.assert_called_once()
+    call_kwargs = background_tasks.add_task.call_args.kwargs
+    assert call_kwargs["action"] == "conversation_turns_read"
+    assert call_kwargs["log_metadata"]["conversation_id"] == str(cid)
+    assert call_kwargs["log_metadata"]["turn_count"] == 1
+
 
 @pytest.mark.asyncio
 async def test_returns_404_when_conversation_not_found():
@@ -65,7 +74,7 @@ async def test_returns_404_when_conversation_not_found():
     key_info = MagicMock()
 
     with pytest.raises(HTTPException) as exc_info:
-        await get_conversation_turns(uuid4(), request, key_info)
+        await get_conversation_turns(uuid4(), request, MagicMock(), key_info)
     assert exc_info.value.status_code == 404
 
 
@@ -86,5 +95,5 @@ async def test_returns_501_for_inmemory_store():
     key_info = MagicMock()
 
     with pytest.raises(HTTPException) as exc_info:
-        await get_conversation_turns(uuid4(), request, key_info)
+        await get_conversation_turns(uuid4(), request, MagicMock(), key_info)
     assert exc_info.value.status_code == 501

vektra-core/src/vektra_core/advanced_pipeline.py

@@ -206,12 +206,14 @@ async def _run_pre_llm_steps(
         list[StepTrace],
         list[SearchResult],
         bool,
+        bool,
         str,
         list[dict[str, str | None]],
     ]:
         """Run steps 0-6 (rewrite through post_retrieval safeguard).
 
-        Returns (steps, filtered_results, no_relevant_context, effective_query, history).
+        Returns (steps, filtered_results, no_relevant_context, safeguard_blocked,
+        effective_query, history).
         """
         steps: list[StepTrace] = []
 
@@ -232,7 +234,7 @@ async def _run_pre_llm_steps(
                 )
             )
             if not sg_pre.allowed:
-                return steps, [], False, query.question, []
+                return steps, [], False, True, query.question, []
         except Exception as exc:
             log.error("pre_query_safeguard_failed", error=str(exc))
             steps.append(
@@ -361,6 +363,7 @@ async def _run_pre_llm_steps(
         )
 
         # Step 6: Post-retrieval safeguard (DEBT-003)
+        safeguard_blocked = False
         if filtered:
             t0 = time.monotonic()
             sg_ctx = SafeguardContext(
@@ -374,6 +377,7 @@ async def _run_pre_llm_steps(
                 )
                 if not sg_result.allowed:
                     filtered = []
+                    safeguard_blocked = True
                 elif sg_result.filtered_ids:
                     excluded = set(sg_result.filtered_ids)
                     filtered = [r for r in filtered if r.chunk_id not in excluded]
@@ -402,7 +406,14 @@ async def _run_pre_llm_steps(
         if not no_relevant_context and not filtered:
             no_relevant_context = True
 
-        return steps, filtered, no_relevant_context, effective_query, history
+        return (
+            steps,
+            filtered,
+            no_relevant_context,
+            safeguard_blocked,
+            effective_query,
+            history,
+        )
 
     def _build_prompt(
         self,
@@ -487,12 +498,15 @@ async def execute(
             steps,
             filtered,
             no_relevant_context,
+            safeguard_blocked,
             _effective_query,
             history,
         ) = await self._run_pre_llm_steps(query)
 
-        # No relevant context -> skip LLM (strict), continue without context (hybrid)
-        if (no_relevant_context or not filtered) and query.grounding_mode != "hybrid":
+        # Safeguard hard block or no relevant context
+        if safeguard_blocked or (
+            (no_relevant_context or not filtered) and query.grounding_mode != "hybrid"
+        ):
             trace = QueryTrace(
                 response_id=response_id,
                 steps=steps,
@@ -502,18 +516,25 @@ async def execute(
                 prompt_version=self._renderer.prompt_version,
                 created_at=datetime.now(UTC),
             )
-            log.info(
-                "query_no_relevant_context",
-                response_id=str(response_id),
-                namespace=query.namespace,
-            )
+            if safeguard_blocked:
+                log.info(
+                    "query_safeguard_blocked",
+                    response_id=str(response_id),
+                    namespace=query.namespace,
+                )
+            else:
+                log.info(
+                    "query_no_relevant_context",
+                    response_id=str(response_id),
+                    namespace=query.namespace,
+                )
             return QueryResponse(
                 response_id=response_id,
                 answer=None,
                 sources=[],
                 conversation_id=query.conversation_id,
-                context_only=not no_relevant_context,
-                no_relevant_context=no_relevant_context,
+                context_only=False,
+                no_relevant_context=no_relevant_context and not safeguard_blocked,
             ), trace
 
         # Step 7: Build prompt
@@ -641,11 +662,14 @@ async def _stream(
             steps,
             filtered,
             no_relevant_context,
+            safeguard_blocked,
             _effective_query,
             history,
         ) = await self._run_pre_llm_steps(query)
 
-        if (no_relevant_context or not filtered) and query.grounding_mode != "hybrid":
+        if safeguard_blocked or (
+            (no_relevant_context or not filtered) and query.grounding_mode != "hybrid"
+        ):
             yield QueryChunk(type="sources", data=[])
             # Emit trace before done (DEBT-002)
             trace = QueryTrace(
@@ -657,6 +681,12 @@ async def _stream(
                 prompt_version=self._renderer.prompt_version,
                 created_at=datetime.now(UTC),
             )
+            if safeguard_blocked:
+                log.info(
+                    "query_safeguard_blocked",
+                    response_id=str(response_id),
+                    namespace=query.namespace,
+                )
             yield QueryChunk(type="trace", data=_trace_to_dict(trace))
             yield QueryChunk(type="done", data="")
             return

vektra-core/src/vektra_core/pipeline.py

@@ -62,10 +62,11 @@ def _history_to_messages(history: list[dict[str, str | None]]) -> list[Message]:
     """Convert conversation history to role-based Message objects."""
     messages: list[Message] = []
     for turn in history:
+        answer = turn.get("answer")
+        if not answer:
+            continue  # Skip turns with no answer to avoid polluting history
         messages.append(Message(role="user", content=turn["question"] or ""))
-        messages.append(
-            Message(role="assistant", content=turn["answer"] or "[No response]")
-        )
+        messages.append(Message(role="assistant", content=answer))
     return messages
 
 

vektra-core/src/vektra_core/templates/context.j2

@@ -1,3 +1,3 @@
 <context>
-{% for chunk in chunks %}<source id="{{ loop.index }}">{{ chunk.text }}</source>
+{% for chunk in chunks %}<source id="{{ loop.index }}">{{ chunk.text | e }}</source>
 {% endfor %}</context>