diff --git a/scripts/eval_e2e.py b/scripts/eval_e2e.py index 9f4ad808..4cda60ae 100644 --- a/scripts/eval_e2e.py +++ b/scripts/eval_e2e.py @@ -115,8 +115,14 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None: return valid = [r for r in results if r.error is None] - answered = sum(1 for r in valid if r.answer is not None) + answered = sum( + 1 for r in valid if r.answer is not None and not r.no_relevant_context + ) + answered_no_ctx = sum( + 1 for r in valid if r.answer is not None and r.no_relevant_context + ) no_context = sum(1 for r in valid if r.no_relevant_context) + unanswered = sum(1 for r in valid if r.answer is None) avg_sources = sum(r.num_sources for r in valid) / evaluated durations = sorted(r.duration_ms for r in valid) @@ -129,8 +135,10 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None: print(f"Questions: {total} ({errors} errors)") print(f"Duration: {elapsed_s:.1f}s total") print("") - print(f"Answered: {answered}/{evaluated} ({answered / evaluated:.0%})") - print(f"No context: {no_context}/{evaluated}") + print(f"Answered (grounded): {answered}/{evaluated} ({answered / evaluated:.0%})") + print(f"Answered (no ctx): {answered_no_ctx}/{evaluated} (LLM without retrieval)") + print(f"Unanswered: {unanswered}/{evaluated}") + print(f"No context: {no_context}/{evaluated}") print(f"Avg sources: {avg_sources:.1f}") print(f"Latency p50: {p50:.0f}ms") print(f"Latency p95: {p95:.0f}ms") @@ -141,9 +149,13 @@ def print_summary(results: list[E2EResult], elapsed_s: float) -> None: print("\nBy category:") for cat in categories: cat_results = [r for r in valid if r.category == cat] - cat_answered = sum(1 for r in cat_results if r.answer is not None) + cat_grounded = sum( + 1 + for r in cat_results + if r.answer is not None and not r.no_relevant_context + ) print( - f" {cat:<15} answered={cat_answered}/{len(cat_results)} n={len(cat_results)}" + f" {cat:<15} grounded={cat_grounded}/{len(cat_results)} n={len(cat_results)}" ) print(f"{'=' * 60}") diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py index e5e35d98..5fa0edf2 100644 --- a/scripts/eval_retrieval.py +++ b/scripts/eval_retrieval.py @@ -2,7 +2,9 @@ """Retrieval-only evaluation harness (TECH-002). Reads a JSONL dataset, calls the Vektra search API for each question, -and computes retrieval quality metrics. No LLM calls are made. +and computes retrieval quality metrics. +By default uses /api/v1/search (no LLM). With --use-pipeline it calls +/api/v1/query which runs the full RAG pipeline including LLM. Usage: python scripts/eval_retrieval.py [OPTIONS] @@ -17,6 +19,7 @@ import os import sys import time +import unicodedata from dataclasses import dataclass, field from pathlib import Path @@ -56,10 +59,23 @@ def load_dataset(path: str) -> list[dict]: return entries +def _normalize(s: str) -> str: + """Lowercase, strip diacritics and apostrophes for keyword matching.""" + nfkd = unicodedata.normalize("NFKD", s.casefold()) + base = "".join(c for c in nfkd if not unicodedata.combining(c)) + for ch in ("'", "\u2019", "\u2018", "\u02bc", "`"): + base = base.replace(ch, "") + return base + + def chunk_matches_keywords(text: str, keywords: list[str]) -> bool: - """Check if a chunk's text contains any of the expected keywords (case-insensitive).""" - text_lower = text.lower() - return any(kw.lower() in text_lower for kw in keywords) + """Check if a chunk's text contains any of the expected keywords. + + Matching is case-insensitive and diacritic-insensitive so that + ASCII dataset entries like "liberta'" match Unicode text "libertà". + """ + text_norm = _normalize(text) + return any(_normalize(kw) in text_norm for kw in keywords) def evaluate_question( @@ -67,11 +83,11 @@ def evaluate_question( entry: dict, top_k: int, search_mode: str, - use_query: bool = False, + use_pipeline: bool = False, ) -> QuestionResult: """Run a single search/query and compute retrieval metrics. - When use_query=True, calls /api/v1/query (includes reranker) and + When use_pipeline=True, calls /api/v1/query (includes reranker) and evaluates on the returned sources. Otherwise calls /api/v1/search (raw vector search, no reranker). """ @@ -83,7 +99,7 @@ def evaluate_question( language = entry.get("language", "unknown") try: - if use_query: + if use_pipeline: resp = client.post( "/api/v1/query", json={ @@ -295,9 +311,9 @@ def main() -> None: help="Search mode (default: hybrid)", ) parser.add_argument( - "--use-query", + "--use-pipeline", action="store_true", - help="Use /api/v1/query (with reranker) instead of /api/v1/search", + help="Use /api/v1/query (full RAG pipeline with reranker + LLM) instead of /api/v1/search (raw vector search). Results reflect the pipeline's source selection, not pure retrieval.", ) args = parser.parse_args() @@ -323,11 +339,13 @@ def main() -> None: print(f"Loaded {len(dataset)} questions from {args.dataset}") mode_info = ( - "query (with reranker)" if args.use_query else f"search mode={args.search_mode}" + "pipeline (reranker + LLM)" + if args.use_pipeline + else f"search mode={args.search_mode}" ) print(f"API: {api_url} top_k={args.top_k} {mode_info}") - timeout = 120.0 if args.use_query else 30.0 + timeout = 120.0 if args.use_pipeline else 30.0 client = httpx.Client( base_url=api_url, headers={"Authorization": f"Bearer {api_key}"}, @@ -338,7 +356,7 @@ def main() -> None: t0 = time.monotonic() for i, entry in enumerate(dataset): result = evaluate_question( - client, entry, args.top_k, args.search_mode, use_query=args.use_query + client, entry, args.top_k, args.search_mode, use_pipeline=args.use_pipeline ) results.append(result) status = "HIT" if result.hit else ("ERR" if result.error else "MISS") diff --git a/tests/eval/dataset.jsonl b/tests/eval/dataset.jsonl index 4ec6b8e1..9f976d6f 100644 --- a/tests/eval/dataset.jsonl +++ b/tests/eval/dataset.jsonl @@ -46,7 +46,7 @@ {"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} {"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} {"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} -{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": ["pena di morte", "non e' ammessa"], "namespace": "default", "category": "factual", "language": "it"} {"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} {"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} {"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} diff --git a/vektra-admin/src/vektra_admin/api.py b/vektra-admin/src/vektra_admin/api.py index 752ab313..60154039 100644 --- a/vektra-admin/src/vektra_admin/api.py +++ b/vektra-admin/src/vektra_admin/api.py @@ -466,6 +466,7 @@ class ConversationTurnDetail(BaseModel): async def get_conversation_turns( conversation_id: UUID, request: Request, + background_tasks: BackgroundTasks, _key: ApiKeyInfo = Depends(require_scope("admin")), ) -> Any: """Return decrypted conversation turns with full metadata (admin only).""" @@ -488,6 +489,24 @@ async def get_conversation_turns( if turns is None: raise HTTPException(status_code=404, detail="Conversation not found") + # Audit log: sensitive content access + request_id = getattr(request.state, "request_id", None) + if request_id: + background_tasks.add_task( + _audit.log_event, + key_id=_key.key_id, + endpoint=f"/api/v1/admin/conversations/{conversation_id}/turns", + method="GET", + status_code=200, + request_id=request_id, + action="conversation_turns_read", + log_metadata={ + "namespace": getattr(request.state, "rls_namespace", None), + "conversation_id": str(conversation_id), + "turn_count": len(turns), + }, + ) + return turns diff --git a/vektra-admin/tests/test_admin_turns.py b/vektra-admin/tests/test_admin_turns.py index ed60a393..97f4e54a 100644 --- a/vektra-admin/tests/test_admin_turns.py +++ b/vektra-admin/tests/test_admin_turns.py @@ -38,14 +38,23 @@ async def test_returns_turns_from_persistent_store(): request = MagicMock() request.app.state = app_state + request.state.request_id = "test-req-id" + background_tasks = MagicMock() key_info = MagicMock() key_info.scopes = ["admin"] - result = await get_conversation_turns(cid, request, key_info) + result = await get_conversation_turns(cid, request, background_tasks, key_info) assert result == turns conv_store.get_turns_detail.assert_called_once_with(cid) + # Verify audit log was queued + background_tasks.add_task.assert_called_once() + call_kwargs = background_tasks.add_task.call_args.kwargs + assert call_kwargs["action"] == "conversation_turns_read" + assert call_kwargs["log_metadata"]["conversation_id"] == str(cid) + assert call_kwargs["log_metadata"]["turn_count"] == 1 + @pytest.mark.asyncio async def test_returns_404_when_conversation_not_found(): @@ -65,7 +74,7 @@ async def test_returns_404_when_conversation_not_found(): key_info = MagicMock() with pytest.raises(HTTPException) as exc_info: - await get_conversation_turns(uuid4(), request, key_info) + await get_conversation_turns(uuid4(), request, MagicMock(), key_info) assert exc_info.value.status_code == 404 @@ -86,5 +95,5 @@ async def test_returns_501_for_inmemory_store(): key_info = MagicMock() with pytest.raises(HTTPException) as exc_info: - await get_conversation_turns(uuid4(), request, key_info) + await get_conversation_turns(uuid4(), request, MagicMock(), key_info) assert exc_info.value.status_code == 501 diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index b392224d..4e1984c5 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -206,12 +206,14 @@ async def _run_pre_llm_steps( list[StepTrace], list[SearchResult], bool, + bool, str, list[dict[str, str | None]], ]: """Run steps 0-6 (rewrite through post_retrieval safeguard). - Returns (steps, filtered_results, no_relevant_context, effective_query, history). + Returns (steps, filtered_results, no_relevant_context, safeguard_blocked, + effective_query, history). """ steps: list[StepTrace] = [] @@ -232,7 +234,7 @@ async def _run_pre_llm_steps( ) ) if not sg_pre.allowed: - return steps, [], False, query.question, [] + return steps, [], False, True, query.question, [] except Exception as exc: log.error("pre_query_safeguard_failed", error=str(exc)) steps.append( @@ -361,6 +363,7 @@ async def _run_pre_llm_steps( ) # Step 6: Post-retrieval safeguard (DEBT-003) + safeguard_blocked = False if filtered: t0 = time.monotonic() sg_ctx = SafeguardContext( @@ -374,6 +377,7 @@ async def _run_pre_llm_steps( ) if not sg_result.allowed: filtered = [] + safeguard_blocked = True elif sg_result.filtered_ids: excluded = set(sg_result.filtered_ids) filtered = [r for r in filtered if r.chunk_id not in excluded] @@ -402,7 +406,14 @@ async def _run_pre_llm_steps( if not no_relevant_context and not filtered: no_relevant_context = True - return steps, filtered, no_relevant_context, effective_query, history + return ( + steps, + filtered, + no_relevant_context, + safeguard_blocked, + effective_query, + history, + ) def _build_prompt( self, @@ -487,12 +498,15 @@ async def execute( steps, filtered, no_relevant_context, + safeguard_blocked, _effective_query, history, ) = await self._run_pre_llm_steps(query) - # No relevant context -> skip LLM (strict), continue without context (hybrid) - if (no_relevant_context or not filtered) and query.grounding_mode != "hybrid": + # Safeguard hard block or no relevant context + if safeguard_blocked or ( + (no_relevant_context or not filtered) and query.grounding_mode != "hybrid" + ): trace = QueryTrace( response_id=response_id, steps=steps, @@ -502,18 +516,25 @@ async def execute( prompt_version=self._renderer.prompt_version, created_at=datetime.now(UTC), ) - log.info( - "query_no_relevant_context", - response_id=str(response_id), - namespace=query.namespace, - ) + if safeguard_blocked: + log.info( + "query_safeguard_blocked", + response_id=str(response_id), + namespace=query.namespace, + ) + else: + log.info( + "query_no_relevant_context", + response_id=str(response_id), + namespace=query.namespace, + ) return QueryResponse( response_id=response_id, answer=None, sources=[], conversation_id=query.conversation_id, - context_only=not no_relevant_context, - no_relevant_context=no_relevant_context, + context_only=False, + no_relevant_context=no_relevant_context and not safeguard_blocked, ), trace # Step 7: Build prompt @@ -641,11 +662,14 @@ async def _stream( steps, filtered, no_relevant_context, + safeguard_blocked, _effective_query, history, ) = await self._run_pre_llm_steps(query) - if (no_relevant_context or not filtered) and query.grounding_mode != "hybrid": + if safeguard_blocked or ( + (no_relevant_context or not filtered) and query.grounding_mode != "hybrid" + ): yield QueryChunk(type="sources", data=[]) # Emit trace before done (DEBT-002) trace = QueryTrace( @@ -657,6 +681,12 @@ async def _stream( prompt_version=self._renderer.prompt_version, created_at=datetime.now(UTC), ) + if safeguard_blocked: + log.info( + "query_safeguard_blocked", + response_id=str(response_id), + namespace=query.namespace, + ) yield QueryChunk(type="trace", data=_trace_to_dict(trace)) yield QueryChunk(type="done", data="") return diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index 7251a2c5..e8d58f6a 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -62,10 +62,11 @@ def _history_to_messages(history: list[dict[str, str | None]]) -> list[Message]: """Convert conversation history to role-based Message objects.""" messages: list[Message] = [] for turn in history: + answer = turn.get("answer") + if not answer: + continue # Skip turns with no answer to avoid polluting history messages.append(Message(role="user", content=turn["question"] or "")) - messages.append( - Message(role="assistant", content=turn["answer"] or "[No response]") - ) + messages.append(Message(role="assistant", content=answer)) return messages diff --git a/vektra-core/src/vektra_core/templates/context.j2 b/vektra-core/src/vektra_core/templates/context.j2 index 9dc944d8..cde635de 100644 --- a/vektra-core/src/vektra_core/templates/context.j2 +++ b/vektra-core/src/vektra_core/templates/context.j2 @@ -1,3 +1,3 @@ -{% for chunk in chunks %}{{ chunk.text }} +{% for chunk in chunks %}{{ chunk.text | e }} {% endfor %} diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index f746d613..bdd1c488 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -38,6 +38,8 @@ def _make_pipeline_config(**overrides) -> QueryPipelineConfig: "VEKTRA_RESPONSE_TOKEN_RESERVE": 512, "VEKTRA_CONTEXT_CHUNK_RATIO": 0.6, "VEKTRA_QUERY_PIPELINE": "advanced", + "VEKTRA_EVAL_MODE": False, + "VEKTRA_DEBUG_LOG_QUERIES": False, } defaults.update(overrides) return QueryPipelineConfig.model_validate(defaults) @@ -404,6 +406,39 @@ async def test_post_retrieval_safeguard_failure_continues(): assert sg_step.metadata.get("skipped") is True +async def test_post_retrieval_safeguard_denied_blocks_llm(): + """When post_retrieval returns allowed=False, LLM is never called (even in hybrid).""" + results = [_make_search_result(0.8, "sensitive content")] + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=results) + + safeguard = _make_safeguard() + safeguard.post_retrieval = AsyncMock(return_value=SafeguardResult(allowed=False)) + + llm = MagicMock() + llm.complete = AsyncMock() + llm.count_tokens = MagicMock(return_value=10) + + pipeline = _make_pipeline( + vector_store=vector_store, + safeguard=safeguard, + llm=llm, + ) + # Use hybrid mode to verify safeguard block overrides grounding mode + response, trace = await pipeline.execute( + QueryRequest(question="test", grounding_mode="hybrid") + ) + + assert response.answer is None + assert response.sources == [] + assert response.no_relevant_context is False # not a context issue + assert response.context_only is False # not context_only either + llm.complete.assert_not_awaited() + + sg_step = next(s for s in trace.steps if s.name == "post_retrieval_safeguard") + assert sg_step.metadata["allowed"] is False + + # --------------------------------------------------------------------------- # Graceful degradation # --------------------------------------------------------------------------- diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index 5e1a5dd3..b3d42807 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -43,6 +43,8 @@ def _make_pipeline_config(**overrides) -> QueryPipelineConfig: "VEKTRA_CHUNK_DEDUP_ENABLED": True, "VEKTRA_RESPONSE_TOKEN_RESERVE": 512, "VEKTRA_CONTEXT_CHUNK_RATIO": 0.6, + "VEKTRA_EVAL_MODE": False, + "VEKTRA_DEBUG_LOG_QUERIES": False, } defaults.update(overrides) return QueryPipelineConfig.model_validate(defaults) diff --git a/vektra-learn/tests/test_trace_persistence.py b/vektra-learn/tests/test_trace_persistence.py index 0c3f83dc..26681d08 100644 --- a/vektra-learn/tests/test_trace_persistence.py +++ b/vektra-learn/tests/test_trace_persistence.py @@ -159,7 +159,7 @@ async def test_learn_store_trace_skipped_when_trace_is_none(): trace = None _store_traces = True - if _store_traces and trace is not None and mock_svc: + if _store_traces and trace is not None: await mock_svc.store_trace(None, trace, namespace="test") mock_svc.store_trace.assert_not_called() diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 9755f3a3..e0642522 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -187,6 +187,8 @@ class QueryPipelineConfig(BaseSettings): ) min_relevance_score: float = Field( 0.15, + ge=0.0, + le=1.0, alias="VEKTRA_MIN_RELEVANCE_SCORE", description="Minimum relevance score for chunk inclusion (ARCH-056). Safety net filter; top-k is the primary control.", ) @@ -197,11 +199,14 @@ class QueryPipelineConfig(BaseSettings): ) response_token_reserve: int = Field( 2048, + ge=1, alias="VEKTRA_RESPONSE_TOKEN_RESERVE", description="Tokens reserved for LLM response generation (ARCH-055).", ) context_chunk_ratio: float = Field( 0.6, + gt=0.0, + lt=1.0, alias="VEKTRA_CONTEXT_CHUNK_RATIO", description="Fraction of context window allocated to retrieved chunks (ARCH-055).", ) @@ -571,6 +576,14 @@ def validate_prompt_grounding_mode(cls, v: str) -> str: ) return v + @model_validator(mode="after") + def validate_eval_mode_not_production(self) -> VektraSettings: + if self.env == "production" and self.eval_mode: + raise ValueError( + "VEKTRA_EVAL_MODE must be disabled in production (captures user text in traces)" + ) + return self + def as_llm_config(self) -> LLMConfig: """Extract LLM-specific sub-config.""" return LLMConfig.model_validate(