From 0b213317d989c7a3190bce8e4e0a3978ccee33e9 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 7 Apr 2026 21:02:29 +0000 Subject: [PATCH 1/6] feat(observability): add reranker scores, fix streaming model name, eval mode plumbing DEBT-014: rerank() returns RerankResult with all_scores for every evaluated candidate (not just top_k). Trace metadata now includes per-chunk reranker and original scores for post-hoc threshold analysis. BUG-019: capture resolved model name from litellm streaming chunks via CompletionChunk.model field. Streaming and non-streaming traces now report the same llm_model value. DEBT-015: when VEKTRA_EVAL_MODE=true, query_rewrite StepTrace includes original_query and rewritten_query text for diagnosing rewrite quality. DEBT-009: when VEKTRA_DEBUG_LOG_QUERIES=true, _rewrite_query() emits a debug-level structlog event with both original and rewritten text. FEAT-019: when VEKTRA_EVAL_MODE=true, build_prompt StepTrace includes the full messages list sent to the LLM for prompt inspection. Both eval_mode and debug_log_queries are added to QueryPipelineConfig (read from env vars), avoiding constructor changes to either pipeline. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/advanced_pipeline.py | 62 +++++++++--- vektra-core/src/vektra_core/pipeline.py | 43 ++++++--- .../vektra_core/providers/litellm_provider.py | 7 +- vektra-core/src/vektra_core/reranker.py | 50 ++++++---- vektra-core/tests/test_litellm_provider.py | 29 ++++++ vektra-core/tests/test_reranker.py | 95 ++++++++++++++----- vektra-shared/src/vektra_shared/config.py | 10 ++ vektra-shared/src/vektra_shared/types.py | 1 + vektra-shared/tests/test_config.py | 16 ++++ 9 files changed, 245 insertions(+), 68 deletions(-) diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index ec9d83bf..304343f4 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -96,6 +96,8 @@ def __init__( self._sparse_embedding = sparse_embedding self._reranker = reranker self._rewrite_enabled = pipeline_config.rewrite.enabled + self._eval_mode = pipeline_config.eval_mode + self._debug_log_queries = pipeline_config.debug_log_queries # -- Helpers (delegating to shared module-level functions) -- @@ -150,15 +152,28 @@ async def _rewrite_query( if not rewritten: rewritten = question + if self._debug_log_queries: + log.debug( + "query_rewritten", + original_query=question, + rewritten_query=rewritten, + history_turns=len(history), + ) + original_hash = hashlib.sha256(question.encode()).hexdigest()[:8] + metadata: dict[str, object] = { + "original_query_hash": original_hash, + "rewritten": True, + "history_turns_used": len(history), + } + if self._eval_mode: + metadata["original_query"] = question + metadata["rewritten_query"] = rewritten + return rewritten, StepTrace( name="query_rewrite", duration_ms=_elapsed_ms(t0), - metadata={ - "original_query_hash": original_hash, - "rewritten": True, - "history_turns_used": len(history), - }, + metadata=metadata, ) except Exception as exc: log.warning("query_rewrite_failed", error=str(exc)) @@ -276,14 +291,26 @@ async def _run_pre_llm_steps( if self._reranker and results: t0 = time.monotonic() try: - results = await self._reranker.rerank( + rerank_result = await self._reranker.rerank( effective_query, results, top_k=query.top_k ) + results = rerank_result.top_k steps.append( StepTrace( name="rerank", duration_ms=_elapsed_ms(t0), - metadata={"after_rerank": len(results)}, + metadata={ + "after_rerank": len(results), + "candidates_evaluated": len(rerank_result.all_scores), + "scores": [ + { + "chunk_id": cid, + "reranker_score": rs, + "original_score": os, + } + for cid, rs, os in rerank_result.all_scores + ], + }, ) ) except Exception as exc: @@ -413,14 +440,19 @@ def _build_prompt( messages.extend(_history_to_messages(selected_history)) messages.append(Message(role="user", content=user_content)) + build_meta: dict[str, object] = { + "prompt_version": self._renderer.prompt_version, + "chunks_in_prompt": len(selected_chunks), + "history_turns_in_prompt": len(selected_history), + } + if self._eval_mode: + build_meta["messages"] = [ + {"role": m.role, "content": m.content} for m in messages + ] step = StepTrace( name="build_prompt", duration_ms=_elapsed_ms(t0), - metadata={ - "prompt_version": self._renderer.prompt_version, - "chunks_in_prompt": len(selected_chunks), - "history_turns_in_prompt": len(selected_history), - }, + metadata=build_meta, ) return messages, selected_chunks, selected_history, step @@ -622,15 +654,18 @@ async def _stream( # Step 8: Stream LLM tokens t0 = time.monotonic() full_answer_parts: list[str] = [] - llm_model = self._llm_config.provider + llm_model: str | None = None try: token_stream = await self._llm.stream( messages, model=self._llm_config.provider ) async for chunk in token_stream: + if chunk.model and llm_model is None: + llm_model = chunk.model if chunk.content: full_answer_parts.append(chunk.content) yield QueryChunk(type="token", data=chunk.content) + llm_model = llm_model or self._llm_config.provider steps.append( StepTrace( name="llm_stream", @@ -640,6 +675,7 @@ async def _stream( ) except Exception as exc: log.warning("stream_llm_failed", error=str(exc)) + llm_model = llm_model or self._llm_config.provider steps.append( StepTrace( name="llm_stream", diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index ce842b07..d8c4a741 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -262,6 +262,8 @@ def __init__( self._conversation_store = conversation_store self._renderer = renderer self._config = pipeline_config + self._eval_mode = pipeline_config.eval_mode + self._debug_log_queries = pipeline_config.debug_log_queries def _count_tokens(self, text: str) -> int: return _count_tokens_impl(self._llm, self._llm_config.provider, text) @@ -498,15 +500,20 @@ async def execute( messages.extend(_history_to_messages(selected_history)) messages.append(Message(role="user", content=user_content)) + build_meta: dict[str, object] = { + "prompt_version": self._renderer.prompt_version, + "chunks_in_prompt": len(selected_chunks), + "history_turns_in_prompt": len(selected_history), + } + if self._eval_mode: + build_meta["messages"] = [ + {"role": m.role, "content": m.content} for m in messages + ] steps.append( StepTrace( name="build_prompt", duration_ms=_elapsed_ms(t0), - metadata={ - "prompt_version": self._renderer.prompt_version, - "chunks_in_prompt": len(selected_chunks), - "history_turns_in_prompt": len(selected_history), - }, + metadata=build_meta, ) ) @@ -798,38 +805,48 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] messages: list[Message] = [Message(role="system", content=system_text)] messages.extend(_history_to_messages(selected_history)) messages.append(Message(role="user", content=user_content)) + stream_build_meta: dict[str, object] = { + "prompt_version": self._renderer.prompt_version, + "chunks_in_prompt": len(selected_chunks), + "history_turns_in_prompt": len(selected_history), + } + if self._eval_mode: + stream_build_meta["messages"] = [ + {"role": m.role, "content": m.content} for m in messages + ] steps.append( StepTrace( name="build_prompt", duration_ms=_elapsed_ms(t0), - metadata={ - "prompt_version": self._renderer.prompt_version, - "chunks_in_prompt": len(selected_chunks), - "history_turns_in_prompt": len(selected_history), - }, + metadata=stream_build_meta, ) ) # Step 5: Stream LLM tokens t0 = time.monotonic() full_answer_parts: list[str] = [] + llm_model_resolved: str | None = None try: token_stream = await self._llm.stream( messages, model=self._llm_config.provider ) async for chunk in token_stream: + if chunk.model and llm_model_resolved is None: + llm_model_resolved = chunk.model if chunk.content: full_answer_parts.append(chunk.content) yield QueryChunk(type="token", data=chunk.content) + llm_model_resolved = llm_model_resolved or self._llm_config.provider steps.append( StepTrace( name="llm_stream", duration_ms=_elapsed_ms(t0), - metadata={"model": self._llm_config.provider}, + metadata={"model": llm_model_resolved}, ) ) except Exception as exc: log.warning("stream_llm_failed", error=str(exc)) + llm_model_resolved = llm_model_resolved or self._llm_config.provider steps.append( StepTrace( name="llm_stream", @@ -845,7 +862,7 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] chunks_retrieved=[ ChunkRef(chunk_id=r.chunk_id, score=r.score) for r in filtered ], - llm_model=self._llm_config.provider, + llm_model=llm_model_resolved, prompt_version=self._renderer.prompt_version, created_at=datetime.now(UTC), ) @@ -915,7 +932,7 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] chunks_retrieved=[ ChunkRef(chunk_id=r.chunk_id, score=r.score) for r in filtered ], - llm_model=self._llm_config.provider, + llm_model=llm_model_resolved, prompt_version=self._renderer.prompt_version, created_at=datetime.now(UTC), ) diff --git a/vektra-core/src/vektra_core/providers/litellm_provider.py b/vektra-core/src/vektra_core/providers/litellm_provider.py index b0ca180e..65a862f9 100644 --- a/vektra-core/src/vektra_core/providers/litellm_provider.py +++ b/vektra-core/src/vektra_core/providers/litellm_provider.py @@ -104,11 +104,16 @@ async def _stream_impl( **self._base_kwargs, **kwargs, ) + resolved_model: str | None = None async for chunk in response: + if resolved_model is None: + resolved_model = getattr(chunk, "model", None) or model delta = chunk.choices[0].delta content = (delta.content or "") if delta else "" finish = chunk.choices[0].finish_reason - yield CompletionChunk(content=content, done=finish is not None) + yield CompletionChunk( + content=content, done=finish is not None, model=resolved_model + ) async def health_check(self) -> HealthStatus: """Probe the primary model with a short completion (5-second timeout).""" diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 9d7e9c70..87db0be4 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -39,6 +39,16 @@ def _sigmoid(x: float) -> float: return z / (1.0 + z) +@dataclasses.dataclass +class RerankResult: + """Reranking output with full score visibility (DEBT-014).""" + + top_k: list[SearchResult] + all_scores: list[ + tuple[str, float, float] + ] # (chunk_id, reranker_score, original_score) + + class RerankerService: """Wraps the rerankers library for scoring and reordering search results.""" @@ -50,14 +60,14 @@ async def rerank( query: str, results: list[SearchResult], top_k: int, - ) -> list[SearchResult]: + ) -> RerankResult: """Rerank search results using the cross-encoder model. Runs inference in a thread (CPU-bound). Returns top_k results - sorted by reranker score. + sorted by reranker score, plus scores for ALL evaluated candidates. """ if not results: - return [] + return RerankResult(top_k=[], all_scores=[]) docs = [r.text_snippet for r in results] @@ -67,26 +77,34 @@ async def rerank( docs=docs, ) - # Extract scores and detect whether normalization is needed. + # Score ALL candidates and detect whether normalization is needed. # FlashRank produces sigmoid scores in [0, 1]; cross-encoder # produces raw logits that can be negative or > 1. - top_items = ranked.results[:top_k] - raw_scores = [float(item.score) for item in top_items] - needs_sigmoid = any(s < 0.0 or s > 1.0 for s in raw_scores) + all_items = ranked.results + all_raw = [float(item.score) for item in all_items] + needs_sigmoid = any(s < 0.0 or s > 1.0 for s in all_raw) + all_scores: list[tuple[str, float, float]] = [] reranked: list[SearchResult] = [] - for item, raw_score in zip(top_items, raw_scores): + + for item in all_items: original = results[item.doc_id] - normalized = _sigmoid(raw_score) if needs_sigmoid else raw_score - reranked.append( - dataclasses.replace( - original, - score=normalized, - original_score=original.score, - ) + raw = float(item.score) + normalized = _sigmoid(raw) if needs_sigmoid else raw + all_scores.append( + (original.chunk_id, round(normalized, 4), round(original.score, 4)) ) - return reranked + if len(reranked) < top_k: + reranked.append( + dataclasses.replace( + original, + score=normalized, + original_score=original.score, + ) + ) + + return RerankResult(top_k=reranked, all_scores=all_scores) def create_reranker(config: RerankConfig) -> RerankerService | None: diff --git a/vektra-core/tests/test_litellm_provider.py b/vektra-core/tests/test_litellm_provider.py index 2a36fa28..d3bb7183 100644 --- a/vektra-core/tests/test_litellm_provider.py +++ b/vektra-core/tests/test_litellm_provider.py @@ -66,6 +66,7 @@ async def _fake_stream_response(*args, **kwargs): chunk.choices[0].delta = MagicMock() chunk.choices[0].delta.content = text chunk.choices[0].finish_reason = finish + chunk.model = "ollama/llama3" yield chunk with patch( @@ -78,10 +79,38 @@ async def _fake_stream_response(*args, **kwargs): assert len(chunks) == 3 assert chunks[0].content == "Hello" assert chunks[0].done is False + assert chunks[0].model == "ollama/llama3" assert chunks[1].content == " world" + assert chunks[1].model == "ollama/llama3" assert chunks[2].done is True +async def test_stream_model_fallback_when_missing(): + """When streaming chunks lack .model, fall back to the passed model param.""" + config = _make_config() + provider = LitellmProvider(config) + messages = [Message(role="user", content="Hi")] + + async def _fake_stream_no_model(*args, **kwargs): + for text, finish in [("hi", None), ("", "stop")]: + chunk = MagicMock(spec=["choices"]) # no .model attribute + chunk.choices = [MagicMock()] + chunk.choices[0].delta = MagicMock() + chunk.choices[0].delta.content = text + chunk.choices[0].finish_reason = finish + yield chunk + + with patch( + "vektra_core.providers.litellm_provider.litellm.acompletion", + new=AsyncMock(return_value=_fake_stream_no_model()), + ): + stream = await provider.stream(messages, model="custom/model") + chunks = [c async for c in stream] + + # Falls back to the model param since chunks have no .model + assert chunks[0].model == "custom/model" + + async def test_health_check_healthy(): config = _make_config() provider = LitellmProvider(config) diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index 55072b54..ca631da4 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -6,6 +6,7 @@ from vektra_core.reranker import ( RerankerService, + RerankResult, _default_model_for_provider, _sigmoid, create_reranker, @@ -31,7 +32,9 @@ def _make_result(score: float, text: str = "some text") -> SearchResult: async def test_rerank_empty_results(): service = RerankerService(ranker=MagicMock()) result = await service.rerank("query", [], top_k=5) - assert result == [] + assert isinstance(result, RerankResult) + assert result.top_k == [] + assert result.all_scores == [] async def test_rerank_returns_top_k_in_order(): @@ -51,11 +54,13 @@ async def test_rerank_returns_top_k_in_order(): mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) service = RerankerService(ranker=mock_ranker) - reranked = await service.rerank("test query", results, top_k=2) + result = await service.rerank("test query", results, top_k=2) - assert len(reranked) == 2 - assert reranked[0].chunk_id == results[2].chunk_id # doc C - assert reranked[1].chunk_id == results[0].chunk_id # doc A + assert len(result.top_k) == 2 + assert result.top_k[0].chunk_id == results[2].chunk_id # doc C + assert result.top_k[1].chunk_id == results[0].chunk_id # doc A + # all_scores includes ALL 3 candidates, not just top_k + assert len(result.all_scores) == 3 mock_ranker.rank.assert_called_once_with( query="test query", docs=["doc A", "doc B", "doc C"] ) @@ -76,14 +81,16 @@ async def test_rerank_propagates_flashrank_scores(): mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) service = RerankerService(ranker=mock_ranker) - reranked = await service.rerank("query", results, top_k=2) + result = await service.rerank("query", results, top_k=2) # Reranker scores replace vector scores - assert reranked[0].score == 0.85 - assert reranked[1].score == 0.40 + assert result.top_k[0].score == 0.85 + assert result.top_k[1].score == 0.40 # Original vector scores preserved - assert reranked[0].original_score == 0.3 - assert reranked[1].original_score == 0.5 + assert result.top_k[0].original_score == 0.3 + assert result.top_k[1].original_score == 0.5 + # all_scores tracks both candidates + assert len(result.all_scores) == 2 async def test_rerank_normalizes_cross_encoder_logits(): @@ -101,17 +108,21 @@ async def test_rerank_normalizes_cross_encoder_logits(): mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) service = RerankerService(ranker=mock_ranker) - reranked = await service.rerank("query", results, top_k=2) + result = await service.rerank("query", results, top_k=2) # Scores normalized via sigmoid - assert reranked[0].score == _sigmoid(3.5) - assert reranked[1].score == _sigmoid(-2.0) + assert result.top_k[0].score == _sigmoid(3.5) + assert result.top_k[1].score == _sigmoid(-2.0) # Normalized scores are in (0, 1) - assert 0.0 < reranked[0].score < 1.0 - assert 0.0 < reranked[1].score < 1.0 + assert 0.0 < result.top_k[0].score < 1.0 + assert 0.0 < result.top_k[1].score < 1.0 # Original scores preserved - assert reranked[0].original_score == 0.6 - assert reranked[1].original_score == 0.4 + assert result.top_k[0].original_score == 0.6 + assert result.top_k[1].original_score == 0.4 + # all_scores contains sigmoid-normalized values (rounded) + assert len(result.all_scores) == 2 + assert result.all_scores[0][1] == round(_sigmoid(3.5), 4) + assert result.all_scores[1][1] == round(_sigmoid(-2.0), 4) async def test_rerank_preserves_metadata(): @@ -132,14 +143,48 @@ async def test_rerank_preserves_metadata(): mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) service = RerankerService(ranker=mock_ranker) - reranked = await service.rerank("q", results, top_k=1) - - assert reranked[0].chunk_id == "chunk-1" - assert reranked[0].document_id == doc_id - assert reranked[0].document_version == 3 - assert reranked[0].metadata == {"page": 5} - assert reranked[0].score == 0.99 - assert reranked[0].original_score == 0.7 + result = await service.rerank("q", results, top_k=1) + + assert result.top_k[0].chunk_id == "chunk-1" + assert result.top_k[0].document_id == doc_id + assert result.top_k[0].document_version == 3 + assert result.top_k[0].metadata == {"page": 5} + assert result.top_k[0].score == 0.99 + assert result.top_k[0].original_score == 0.7 + + +async def test_all_scores_includes_items_beyond_top_k(): + """all_scores contains entries for ALL candidates, not just top_k (DEBT-014).""" + results = [_make_result(0.5 + i * 0.05, f"doc {i}") for i in range(5)] + + ranked_items = [ + SimpleNamespace(doc_id=4, score=0.95), + SimpleNamespace(doc_id=3, score=0.80), + SimpleNamespace(doc_id=2, score=0.60), + SimpleNamespace(doc_id=1, score=0.30), + SimpleNamespace(doc_id=0, score=0.10), + ] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + result = await service.rerank("query", results, top_k=2) + + # top_k has only 2 results + assert len(result.top_k) == 2 + assert result.top_k[0].chunk_id == results[4].chunk_id + assert result.top_k[1].chunk_id == results[3].chunk_id + + # all_scores has ALL 5 candidates + assert len(result.all_scores) == 5 + # Scores are in descending order (reranker order) + scores = [s[1] for s in result.all_scores] + assert scores == sorted(scores, reverse=True) + # Each entry is (chunk_id, reranker_score, original_score) + for chunk_id, reranker_score, original_score in result.all_scores: + assert isinstance(chunk_id, str) + assert 0.0 <= reranker_score <= 1.0 + assert 0.0 <= original_score <= 1.0 # --------------------------------------------------------------------------- diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 515e5328..9755f3a3 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -215,6 +215,16 @@ class QueryPipelineConfig(BaseSettings): alias="VEKTRA_PROMPT_GROUNDING_MODE", description="Prompt grounding mode: 'strict' (context + history only) or 'hybrid' (fallback to training data when confident).", ) + eval_mode: bool = Field( + False, + alias="VEKTRA_EVAL_MODE", + description="Capture query/prompt text in traces for batch evaluation. Staging only.", + ) + debug_log_queries: bool = Field( + False, + alias="VEKTRA_DEBUG_LOG_QUERIES", + description="Log original and rewritten query text at debug level. Development only.", + ) rewrite: RewriteConfig = Field(default_factory=RewriteConfig) rerank: RerankConfig = Field(default_factory=RerankConfig) diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index 56f599bc..3f25d42a 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -212,6 +212,7 @@ class CompletionChunk: content: str done: bool = False + model: str | None = None @dataclass diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 79eea8c2..9a3dd1fd 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -65,6 +65,22 @@ def test_grounding_mode_invalid(self) -> None: with pytest.raises(ValidationError, match="grounding_mode"): QueryPipelineConfig(VEKTRA_PROMPT_GROUNDING_MODE="invalid") + def test_eval_mode_default_false(self) -> None: + cfg = QueryPipelineConfig() + assert cfg.eval_mode is False + + def test_debug_log_queries_default_false(self) -> None: + cfg = QueryPipelineConfig() + assert cfg.debug_log_queries is False + + def test_eval_mode_from_env(self) -> None: + cfg = QueryPipelineConfig(VEKTRA_EVAL_MODE=True) + assert cfg.eval_mode is True + + def test_debug_log_queries_from_env(self) -> None: + cfg = QueryPipelineConfig(VEKTRA_DEBUG_LOG_QUERIES=True) + assert cfg.debug_log_queries is True + class TestRewriteConfig: def test_defaults(self) -> None: From 00aaed8eb8c9037bf44388952a0726191eb0714c Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 7 Apr 2026 21:09:12 +0000 Subject: [PATCH 2/6] test(observability): add eval_mode gating tests and fix os shadowing Add 4 tests for DEBT-015/FEAT-019 eval_mode gating: - rewritten query captured in trace when eval_mode=True - rewritten query excluded when eval_mode=False - full prompt messages captured when eval_mode=True - messages excluded when eval_mode=False Fix H1: rename `os` loop variable to `orig` in rerank scores comprehension to avoid shadowing the builtin. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/advanced_pipeline.py | 4 +- vektra-core/tests/test_advanced_pipeline.py | 133 ++++++++++++++++++ 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 304343f4..64b40b78 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -306,9 +306,9 @@ async def _run_pre_llm_steps( { "chunk_id": cid, "reranker_score": rs, - "original_score": os, + "original_score": orig, } - for cid, rs, os in rerank_result.all_scores + for cid, rs, orig in rerank_result.all_scores ], }, ) diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index c39c1ee9..acc1d1e7 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -533,3 +533,136 @@ async def test_stream_no_relevant_context_still_emits_trace(): types = [c.type for c in chunks] assert "trace" in types assert "done" in types + + +# --------------------------------------------------------------------------- +# Eval mode / debug logging (DEBT-015, DEBT-009, FEAT-019) +# --------------------------------------------------------------------------- + + +async def test_eval_mode_captures_rewritten_query(): + """With eval_mode=True, query_rewrite trace includes query text (DEBT-015).""" + conv_store = InMemoryConversationStore(max_turns=10) + cid = uuid4() + await conv_store.add_turn( + cid, "What is RAG?", "RAG is retrieval-augmented generation." + ) + + llm = MagicMock() + llm.count_tokens = MagicMock(return_value=10) + rewrite_resp = CompletionResponse( + content="What is retrieval-augmented generation (RAG)?", + model="m", + prompt_tokens=10, + completion_tokens=10, + total_tokens=20, + ) + answer_resp = CompletionResponse( + content="RAG combines retrieval and generation.", + model="m", + prompt_tokens=50, + completion_tokens=20, + total_tokens=70, + ) + llm.complete = AsyncMock(side_effect=[rewrite_resp, answer_resp]) + + results = [_make_search_result(0.8, "RAG context")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + llm=llm, + vector_store=vs, + conversation_store=conv_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + _, trace = await pipeline.execute( + QueryRequest(question="Tell me more", conversation_id=cid) + ) + + rewrite_step = next(s for s in trace.steps if s.name == "query_rewrite") + assert rewrite_step.metadata["rewritten"] is True + assert "rewritten_query" in rewrite_step.metadata + assert "original_query" in rewrite_step.metadata + assert rewrite_step.metadata["original_query"] == "Tell me more" + + +async def test_eval_mode_off_excludes_query_text(): + """With eval_mode=False (default), no query text in rewrite trace.""" + conv_store = InMemoryConversationStore(max_turns=10) + cid = uuid4() + await conv_store.add_turn(cid, "Hello", "Hi there.") + + llm = MagicMock() + llm.count_tokens = MagicMock(return_value=10) + rewrite_resp = CompletionResponse( + content="Rewritten query", + model="m", + prompt_tokens=10, + completion_tokens=10, + total_tokens=20, + ) + answer_resp = CompletionResponse( + content="Answer.", + model="m", + prompt_tokens=50, + completion_tokens=20, + total_tokens=70, + ) + llm.complete = AsyncMock(side_effect=[rewrite_resp, answer_resp]) + + results = [_make_search_result(0.8, "context")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + llm=llm, + vector_store=vs, + conversation_store=conv_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=False), + ) + _, trace = await pipeline.execute( + QueryRequest(question="Follow up", conversation_id=cid) + ) + + rewrite_step = next(s for s in trace.steps if s.name == "query_rewrite") + assert rewrite_step.metadata["rewritten"] is True + assert "rewritten_query" not in rewrite_step.metadata + assert "original_query" not in rewrite_step.metadata + + +async def test_eval_mode_captures_prompt_messages(): + """With eval_mode=True, build_prompt trace includes full messages (FEAT-019).""" + results = [_make_search_result(0.8, "context about RAG")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + vector_store=vs, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + _, trace = await pipeline.execute(QueryRequest(question="What is RAG?")) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + assert "messages" in build_step.metadata + msgs = build_step.metadata["messages"] + assert isinstance(msgs, list) + assert len(msgs) >= 2 # system + user at minimum + assert msgs[0]["role"] == "system" + assert msgs[-1]["role"] == "user" + + +async def test_eval_mode_off_excludes_prompt_messages(): + """With eval_mode=False (default), no messages in build_prompt trace.""" + results = [_make_search_result(0.8, "context")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + vector_store=vs, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=False), + ) + _, trace = await pipeline.execute(QueryRequest(question="test")) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + assert "messages" not in build_step.metadata From cf7162a45398a8962e5e655a95d1b221c7a01a46 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 7 Apr 2026 21:29:39 +0000 Subject: [PATCH 3/6] style(review): address PR #55 review comments - Gate reranker scores behind eval_mode to avoid trace bloat (Gemini #2) - Add model to failed llm_stream StepTrace in both pipelines (CodeRabbit #4) - Fix test_reranking_narrows_results mock to return RerankResult (CodeRabbit #5) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/advanced_pipeline.py | 28 ++++++++++--------- vektra-core/src/vektra_core/pipeline.py | 2 +- vektra-core/tests/test_advanced_pipeline.py | 6 ++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 64b40b78..ce11ff22 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -295,22 +295,24 @@ async def _run_pre_llm_steps( effective_query, results, top_k=query.top_k ) results = rerank_result.top_k + rerank_meta: dict[str, object] = { + "after_rerank": len(results), + "candidates_evaluated": len(rerank_result.all_scores), + } + if self._eval_mode: + rerank_meta["scores"] = [ + { + "chunk_id": cid, + "reranker_score": rs, + "original_score": orig, + } + for cid, rs, orig in rerank_result.all_scores + ] steps.append( StepTrace( name="rerank", duration_ms=_elapsed_ms(t0), - metadata={ - "after_rerank": len(results), - "candidates_evaluated": len(rerank_result.all_scores), - "scores": [ - { - "chunk_id": cid, - "reranker_score": rs, - "original_score": orig, - } - for cid, rs, orig in rerank_result.all_scores - ], - }, + metadata=rerank_meta, ) ) except Exception as exc: @@ -680,7 +682,7 @@ async def _stream( StepTrace( name="llm_stream", duration_ms=_elapsed_ms(t0), - metadata={"error": str(exc)}, + metadata={"model": llm_model, "error": str(exc)}, ) ) yield QueryChunk(type="error", data="LLM unavailable") diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index d8c4a741..7251a2c5 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -851,7 +851,7 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] StepTrace( name="llm_stream", duration_ms=_elapsed_ms(t0), - metadata={"error": str(exc)}, + metadata={"model": llm_model_resolved, "error": str(exc)}, ) ) yield QueryChunk(type="error", data="LLM unavailable") diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index acc1d1e7..0db82b4d 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -5,7 +5,7 @@ from vektra_core.advanced_pipeline import AdvancedQueryPipeline from vektra_core.conversation import InMemoryConversationStore -from vektra_core.reranker import RerankerService +from vektra_core.reranker import RerankerService, RerankResult from vektra_core.templates import TemplateRenderer from vektra_shared.config import LLMConfig, QueryPipelineConfig from vektra_shared.types import ( @@ -284,7 +284,9 @@ async def test_reranking_narrows_results(): # Mock reranker to return only the best result reranker = AsyncMock(spec=RerankerService) - reranker.rerank = AsyncMock(return_value=[results[2]]) + reranker.rerank = AsyncMock( + return_value=RerankResult(top_k=[results[2]], all_scores=[]) + ) pipeline = _make_pipeline(vector_store=vector_store, reranker=reranker) response, trace = await pipeline.execute(QueryRequest(question="test", top_k=1)) From 0e4460c24e463a7437a685d845a2052ed24c8f97 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 8 Apr 2026 13:09:10 +0000 Subject: [PATCH 4/6] fix(observability): stabilize eval_mode metadata across all rewrite paths Add original_query and rewritten_query to skip and error paths in _rewrite_query() when eval_mode is True. Previously only the success path included these fields, making eval traces incomplete for skipped or failed rewrites. Also populate all_scores in test_reranking_narrows_results mock for realistic trace metadata coverage. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/advanced_pipeline.py | 18 ++++++++++++++++-- vektra-core/tests/test_advanced_pipeline.py | 16 ++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index ce11ff22..b392224d 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -126,10 +126,17 @@ async def _rewrite_query( t0 = time.monotonic() if not self._rewrite_enabled or not history: + skip_meta: dict[str, object] = { + "rewritten": False, + "history_turns_used": 0, + } + if self._eval_mode: + skip_meta["original_query"] = question + skip_meta["rewritten_query"] = question return question, StepTrace( name="query_rewrite", duration_ms=_elapsed_ms(t0), - metadata={"rewritten": False, "history_turns_used": 0}, + metadata=skip_meta, ) try: @@ -177,10 +184,17 @@ async def _rewrite_query( ) except Exception as exc: log.warning("query_rewrite_failed", error=str(exc)) + err_meta: dict[str, object] = { + "rewritten": False, + "error": str(exc), + } + if self._eval_mode: + err_meta["original_query"] = question + err_meta["rewritten_query"] = question return question, StepTrace( name="query_rewrite", duration_ms=_elapsed_ms(t0), - metadata={"rewritten": False, "error": str(exc)}, + metadata=err_meta, ) # -- Pre-LLM steps shared between execute() and execute_stream() -- diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index 0db82b4d..e7e61182 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -282,10 +282,17 @@ async def test_reranking_narrows_results(): vector_store = AsyncMock() vector_store.search = AsyncMock(return_value=results) - # Mock reranker to return only the best result + # Mock reranker to return only the best result, with scores for all candidates reranker = AsyncMock(spec=RerankerService) reranker.rerank = AsyncMock( - return_value=RerankResult(top_k=[results[2]], all_scores=[]) + return_value=RerankResult( + top_k=[results[2]], + all_scores=[ + (results[2].chunk_id, 0.95, 0.9), + (results[0].chunk_id, 0.60, 0.6), + (results[1].chunk_id, 0.20, 0.5), + ], + ) ) pipeline = _make_pipeline(vector_store=vector_store, reranker=reranker) @@ -293,8 +300,9 @@ async def test_reranking_narrows_results(): reranker.rerank.assert_awaited_once() assert len(response.sources) == 1 - step_names = [s.name for s in trace.steps] - assert "rerank" in step_names + rerank_step = next(s for s in trace.steps if s.name == "rerank") + assert rerank_step.metadata["after_rerank"] == 1 + assert rerank_step.metadata["candidates_evaluated"] == 3 async def test_reranking_fallback_on_failure(): From 8ef3c7677270bbec9bc40a2413c1a20e700a43a9 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Thu, 9 Apr 2026 13:09:26 +0000 Subject: [PATCH 5/6] test(observability): add eval_mode gating tests for skip and error rewrite paths Cover the two rewrite paths that were missing eval_mode test coverage: - skip path (no history): verify original_query and rewritten_query present - error path (LLM failure): verify both fields present alongside error Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/tests/test_advanced_pipeline.py | 58 +++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index e7e61182..2de26902 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -641,6 +641,64 @@ async def test_eval_mode_off_excludes_query_text(): assert "original_query" not in rewrite_step.metadata +async def test_eval_mode_skipped_rewrite_includes_query(): + """With eval_mode=True and no history, skip path still includes query text.""" + results = [_make_search_result(0.8, "context")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + vector_store=vs, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + # No conversation_id = no history = rewrite skipped + _, trace = await pipeline.execute(QueryRequest(question="What is RAG?")) + + rewrite_step = next(s for s in trace.steps if s.name == "query_rewrite") + assert rewrite_step.metadata["rewritten"] is False + assert rewrite_step.metadata["original_query"] == "What is RAG?" + assert rewrite_step.metadata["rewritten_query"] == "What is RAG?" + + +async def test_eval_mode_failed_rewrite_includes_query(): + """With eval_mode=True and rewrite failure, error path still includes query text.""" + conv_store = InMemoryConversationStore(max_turns=10) + cid = uuid4() + await conv_store.add_turn(cid, "Hello", "Hi there.") + + llm = MagicMock() + llm.count_tokens = MagicMock(return_value=10) + # First call (rewrite) fails, second call (answer) succeeds + answer_resp = CompletionResponse( + content="Answer.", + model="m", + prompt_tokens=50, + completion_tokens=20, + total_tokens=70, + ) + llm.complete = AsyncMock(side_effect=[RuntimeError("rewrite failed"), answer_resp]) + + results = [_make_search_result(0.8, "context")] + vs = AsyncMock() + vs.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + llm=llm, + vector_store=vs, + conversation_store=conv_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + _, trace = await pipeline.execute( + QueryRequest(question="Follow up", conversation_id=cid) + ) + + rewrite_step = next(s for s in trace.steps if s.name == "query_rewrite") + assert rewrite_step.metadata["rewritten"] is False + assert "error" in rewrite_step.metadata + assert rewrite_step.metadata["original_query"] == "Follow up" + assert rewrite_step.metadata["rewritten_query"] == "Follow up" + + async def test_eval_mode_captures_prompt_messages(): """With eval_mode=True, build_prompt trace includes full messages (FEAT-019).""" results = [_make_search_result(0.8, "context about RAG")] From 09f14cb0428c46eeb2217d683aad9cc87c0d1c53 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Thu, 9 Apr 2026 13:17:52 +0000 Subject: [PATCH 6/6] docs(backlog): update status for items completed in PRs #53-#55 Mark 10 backlog items as completed with PR references: - PR #53: BUG-013, DEBT-011 - PR #54: BUG-020, FEAT-020, DEBT-016 - PR #55: DEBT-014, BUG-019, DEBT-015, DEBT-009, FEAT-019 Also add content assertion to eval_mode prompt test per review. Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 22 ++++++++++----------- vektra-core/tests/test_advanced_pipeline.py | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 5b1ad9db..fa32efba 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -1,6 +1,6 @@ # Vektra Backlog -**Updated**: 2026-03-28 +**Updated**: 2026-04-09 **Format**: Single markdown file for tracking work items --- @@ -24,7 +24,7 @@ ### BUG-020: System prompt "use only this material" conflicts with multi-turn history -**Status**: planned | **Priority**: high | **Created**: 2026-03-28 +**Status**: completed | **Priority**: high | **Created**: 2026-03-28 | **Completed**: 2026-04-04 | **PR**: #54 **Context**: the system prompt instructs the LLM to "Use only this material to answer", where "material" refers to the `` tags in the current user message. In multi-turn conversations, the conversation history is injected as separate user/assistant message pairs *before* the current message. The LLM correctly interprets the rule as applying only to the current `` and ignores information from its own previous answers. @@ -93,7 +93,7 @@ Design considerations: ### FEAT-019: Full prompt observability in eval mode -**Status**: planned | **Priority**: low | **Created**: 2026-03-28 +**Status**: completed | **Priority**: low | **Created**: 2026-03-28 | **Completed**: 2026-04-07 | **PR**: #55 **Related**: useful for diagnosing BUG-020 but not a fix for it. **Context**: when diagnosing RAG behavior, the assembled prompt (system + history + context + question) is the most important artifact, but it is never persisted. The `build_prompt` trace step records chunk count and history turn count, but not the actual text. Without seeing the full prompt, it is impossible to understand why the LLM produced a specific answer (e.g., was Art. 33 in the context? how was the history formatted? did the token budget truncate anything?). @@ -115,7 +115,7 @@ Related to DEBT-015 (rewritten query in eval mode) but broader scope: this captu ### FEAT-020: Configurable prompt grounding mode (strict/hybrid) -**Status**: in_progress | **Priority**: high | **Created**: 2026-03-28 | **Branch**: feat/prompt-grounding-mode +**Status**: completed | **Priority**: high | **Created**: 2026-03-28 | **Completed**: 2026-04-09 | **PR**: #54 **Blocks**: BUG-020 (this implements the fix) **Research**: `vektra-internal/stack/20260328-rag-prompt-research-multi-turn.md` @@ -301,7 +301,7 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. ### BUG-013: QueryTrace not persisted to database -**Status**: in_progress | **Priority**: high | **Created**: 2026-03-23 | **Branch**: fix/query-trace-observability +**Status**: completed | **Priority**: high | **Created**: 2026-03-23 | **Completed**: 2026-04-04 | **PR**: #53 **Analysis**: `vektra-internal/stack/20260323-rag-prompt-chunk-confusion-analysis.md` **Context**: `AnalyticsService.store_trace()` exists and is tested but is never called by any pipeline or endpoint. The `query_traces` table is always empty. Traces are generated by all pipelines (SimpleQueryPipeline, AdvancedQueryPipeline) and emitted via SSE to the client, but discarded server-side. This makes post-hoc diagnosis of query failures impossible - as demonstrated when a multi-turn failure ("si, entrambi") could not be investigated because all diagnostic data was lost. @@ -474,7 +474,7 @@ A client using SSE without generating its own `conversation_id` cannot discover ### DEBT-011: Conversation and query trace observability gaps -**Status**: in_progress | **Priority**: medium | **Created**: 2026-03-25 | **Branch**: fix/query-trace-observability +**Status**: completed | **Priority**: medium | **Created**: 2026-03-25 | **Completed**: 2026-04-04 | **PR**: #53 **Related**: BUG-018 (SSE conversation_id) **Context**: Diagnosing a conversation (`5bf50682`, namespace `ita-100`) revealed multiple observability gaps that make post-hoc analysis of query behavior difficult: @@ -506,7 +506,7 @@ A client using SSE without generating its own `conversation_id` cannot discover ### BUG-019: llm_model field inconsistent between streaming and non-streaming traces -**Status**: planned | **Priority**: low | **Created**: 2026-03-28 +**Status**: completed | **Priority**: low | **Created**: 2026-03-28 | **Completed**: 2026-04-07 | **PR**: #55 **Context**: QueryTrace `llm_model` field has different values depending on the execution path. Non-streaming `execute()` sets it from the return value of `_call_llm_with_fallback()`, which returns the litellm-resolved model name (e.g. `qwen35-27b`). Streaming `_stream()` sets it from `self._llm_config.provider` (raw config value, e.g. `openai/qwen35-27b`). This inconsistency affects trace queries and metrics aggregation by model. @@ -583,7 +583,7 @@ Also consider making `_REWRITE_TOP_K=20` configurable or deriving it from the re ### DEBT-014: Include all reranker scores in QueryTrace (not just post-threshold) -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 +**Status**: completed | **Priority**: medium | **Created**: 2026-03-28 | **Completed**: 2026-04-07 | **PR**: #55 **Context**: `chunks_retrieved` in QueryTrace contains only the chunks that pass the relevance threshold filter. Chunks scored by the reranker but filtered out are lost - there is no record of their chunk_id or score. This makes it impossible to evaluate whether the threshold is too aggressive (cutting good chunks) or too permissive without re-running the query. @@ -602,7 +602,7 @@ The `StepTrace` metadata for the `rerank` step only contains `after_rerank: N` ( ### DEBT-015: Persist rewritten query text in QueryTrace (dev/eval mode only) -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 +**Status**: completed | **Priority**: medium | **Created**: 2026-03-28 | **Completed**: 2026-04-07 | **PR**: #55 **Context**: when query rewriting is active, `_rewrite_query()` produces a rewritten query that replaces the original for embedding and retrieval. The rewritten text is not stored anywhere - the `query_rewrite` step metadata contains only `rewritten: true/false`, `history_turns_used`, and `original_query_hash`. Without the rewritten text, it is impossible to understand why the retrieval returned certain chunks in a multi-turn conversation. @@ -623,7 +623,7 @@ DEBT-009 addresses debug logging of the rewritten query to structlog. This entry ### DEBT-009: Debug logging for rewritten queries -**Status**: planned | **Priority**: medium | **Created**: 2026-03-23 +**Status**: completed | **Priority**: medium | **Created**: 2026-03-23 | **Completed**: 2026-04-07 | **PR**: #55 **Analysis**: `vektra-internal/stack/20260323-rag-prompt-chunk-confusion-analysis.md` **Context**: the `_rewrite_query()` method in AdvancedQueryPipeline does not log the rewritten query text. Only a SHA-256 hash of the original query is stored in StepTrace metadata. This is by design for GDPR (ARCH-041: "QueryTrace does not contain query text or response content"), but makes it impossible to diagnose rewrite failures in development. @@ -642,7 +642,7 @@ DEBT-009 addresses debug logging of the rewritten query to structlog. This entry ### DEBT-016: Remove unused conversation.j2 template and render_conversation() -**Status**: in_progress | **Priority**: low | **Created**: 2026-03-28 | **Branch**: feat/prompt-grounding-mode +**Status**: completed | **Priority**: low | **Created**: 2026-03-28 | **Completed**: 2026-04-04 | **PR**: #54 **Context**: ARCH-054 designed three composable Jinja2 templates: `system.j2`, `context.j2`, `conversation.j2`. During Phase 1 implementation (Wave 3, commit 7939b22), the pipeline chose to pass history as native chat messages via `_history_to_messages()` (user/assistant role pairs) instead of rendering it as text via `conversation.j2`. This is the correct approach for modern chat models. diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index 2de26902..f746d613 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -718,6 +718,7 @@ async def test_eval_mode_captures_prompt_messages(): assert len(msgs) >= 2 # system + user at minimum assert msgs[0]["role"] == "system" assert msgs[-1]["role"] == "user" + assert any(msg.get("content") for msg in msgs) async def test_eval_mode_off_excludes_prompt_messages():