From 044900566e2328801b13c5ee7de811af2d8c312f Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:04:54 +0000 Subject: [PATCH 01/33] fix(core): register conversation store in provider registry The PersistentConversationStore was initialized but never registered in the ProviderRegistry, causing GET /api/v1/conversations/{id} to return 503 "not configured" even when VEKTRA_CONVERSATION_KEY was set. Part of BUG-014. Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-app/src/vektra_app/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vektra-app/src/vektra_app/main.py b/vektra-app/src/vektra_app/main.py index 2c13666c..6fd3e9f0 100644 --- a/vektra-app/src/vektra_app/main.py +++ b/vektra-app/src/vektra_app/main.py @@ -239,6 +239,8 @@ async def _step_5_register_providers( persistence="disabled", ) + registry.register("conversation_store", "default", conversation_store) + # --- Reranker (Phase 2, conditional) --- from vektra_core.reranker import create_reranker From 8d1360043ab4d97956c6b22c01c99279a3c29209 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:05:00 +0000 Subject: [PATCH 02/33] chore(backlog): add BUG-014 conversation persistence gap Tracks the missing create_conversation() call that causes PersistentConversationStore to silently discard all turns. Annotates BUG-010 as partially incomplete. Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index efcb317b..9b1b2545 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -81,6 +81,32 @@ --- +### BUG-014: Conversation rows never created — persistent store silently discards all turns + +**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Analysis**: `vektra-internal/stack/20260324-conversation-persistence-gap-analysis.md` +**Reopens**: BUG-010 (marked completed but acceptance criteria #2 not satisfied) + +**Context**: `PersistentConversationStore.create_conversation()` exists (conversation.py:123-148) but is never called. The learn endpoint generates a `conversation_id` UUID (BUG-010 fix) and passes it to the pipeline, but no `ConversationOrm` row is created in the database. When the pipeline calls `add_turn()`, it looks up the conversation row, finds nothing, logs `conversation_not_found` at warning level, and returns silently. Result: **zero turns are ever persisted**, multi-turn context is broken (get_history returns empty), and `GET /api/v1/conversations/{id}` returns 404. + +**Root cause**: `create_conversation()` requires `namespace_id` and `key_id`, which are available in the API layer but not in the pipeline. `QueryRequest` does not carry auth context. The implementation plan (20260301-core-conversations.md) stated the pipeline should call `create_conversation()`, but the pipeline was never given the required parameters. BUG-010 was closed after adding UUID generation without completing the DB creation step. + +**Additional finding (fixed)**: `PersistentConversationStore` was not registered in the `ProviderRegistry`, so `GET /api/v1/conversations/{id}` returned 503 even when the store was initialized. Fixed by adding `registry.register("conversation_store", "default", conversation_store)` in main.py. + +**Traceability**: REQ-049, ARCH-031, BUG-010, FEAT-004 (blocked by this) + +**Acceptance criteria**: +- [ ] `POST /api/v1/query`: when `conversation_id` is None, create `ConversationOrm` row with namespace_id and key_id, set ID on request +- [ ] `POST /api/v1/query`: when `conversation_id` is provided but row doesn't exist, create it (first-use from client-generated ID) +- [ ] `POST /api/v1/learn/query`: same behavior, deriving key_id from learn service context +- [ ] `add_turn()` successfully persists turns after conversation creation +- [ ] `get_history()` returns previous turns for multi-turn queries +- [ ] `GET /api/v1/conversations/{id}` returns conversation metadata +- [ ] Verified: `conversations` and `conversation_turns` tables populated after widget queries +- [ ] Pipeline code unchanged (no auth context leaking into QueryRequest) + +--- + ### DEBT-009: Debug logging for rewritten queries **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 @@ -1003,7 +1029,7 @@ The `AdvancedQueryPipeline` correctly calls `SparseEmbeddingProvider.embed_query ### BUG-010: ~~Learn query endpoint does not auto-create conversation on first query~~ -**Status**: completed | **Priority**: high | **Created**: 2026-03-16 | **Completed**: 2026-03-22 (v0.3.0) +**Status**: completed (partial — DB row creation missing, tracked as BUG-014) | **Priority**: high | **Created**: 2026-03-16 | **Completed**: 2026-03-22 (v0.3.0) **Origin**: Moodle integration testing (2026-03-16) **Context**: The learn query endpoint (`POST /api/v1/learn/query`) passes `conversation_id` through to the pipeline unchanged. When the widget sends the first query without a `conversation_id` (which is the normal flow), the pipeline receives `None`, skips history retrieval and turn saving, and returns `conversation_id: null`. The widget receives `null` and has nothing to save — so the second query also has no `conversation_id`. Result: **every query is a single-turn query with no conversation continuity**. From fb8b500d4bfc0377479dabe6365135660892f517 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:05:06 +0000 Subject: [PATCH 03/33] docs(claude): add API interaction directive Require consulting docs/reference/api.md or /openapi.json before constructing any API call to prevent auth, parameter, and endpoint mistakes. Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/CLAUDE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 83aecd57..73cff20e 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -2,6 +2,16 @@ @../.s2s/CONTEXT.md +## API interaction + +Before making **any** API call (curl, httpie, scripts), consult `docs/reference/api.md` or the live OpenAPI spec at `/openapi.json` to verify: +- Authentication method and header format +- Parameter names, types, and whether they are query, path, or body params +- Request body field names (e.g. `question` vs `query`) +- Correct endpoint for the task (e.g. `/api/v1/query` for RAG, `/api/v1/search` for raw vector search) + +Do not construct API calls from memory or guesswork. + ## Spec2Ship Commands - `/s2s:specs` - Define requirements via roundtable From d965db580228198fbc1487583714d4e57566c519 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:12:04 +0000 Subject: [PATCH 04/33] fix(core): create conversation row before first turn (BUG-014) PersistentConversationStore.add_turn() requires an existing ConversationOrm row, but no one was creating it. All turns were silently discarded and multi-turn context was broken. Changes: - Add ensure_conversation() to PersistentConversationStore (upsert that creates the row only if it doesn't exist) - Extend create_conversation() to accept an optional conversation_id - Core API: create conversation before calling pipeline, using namespace_id and key_id from the auth layer - Learn API: ensure conversation exists using sentinel key_id (JWT auth has no API key) Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/src/vektra_core/api.py | 26 ++++++++++- vektra-core/src/vektra_core/conversation.py | 49 +++++++++++++++++---- vektra-learn/src/vektra_learn/api.py | 28 ++++++++++-- 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/vektra-core/src/vektra_core/api.py b/vektra-core/src/vektra_core/api.py index 471d9eae..eeb77e87 100644 --- a/vektra-core/src/vektra_core/api.py +++ b/vektra-core/src/vektra_core/api.py @@ -208,10 +208,34 @@ async def query( accept = request.headers.get("accept", "") use_stream = body.stream or "text/event-stream" in accept + # Ensure conversation row exists for persistent multi-turn (BUG-014). + # The API layer creates the row because it has namespace_id and key_id, + # which the pipeline does not (and should not) receive. + conversation_id = body.conversation_id + try: + conv_store = registry.get("conversation_store", "default") + if isinstance(conv_store, PersistentConversationStore): + if conversation_id is None: + conversation_id = await conv_store.create_conversation( + namespace_id=body.namespace, + key_id=_key.key_id, + ) + else: + # Client-provided ID: create row if it doesn't exist yet. + await conv_store.ensure_conversation( + conversation_id=conversation_id, + namespace_id=body.namespace, + key_id=_key.key_id, + ) + except ValueError: + pass # conversation store not registered (optional) + except Exception as exc: + log.warning("conversation_create_failed", error=str(exc)) + query_req = QueryRequest( question=body.question, namespace=body.namespace, - conversation_id=body.conversation_id, + conversation_id=conversation_id, top_k=body.top_k, stream=use_stream, ) diff --git a/vektra-core/src/vektra_core/conversation.py b/vektra-core/src/vektra_core/conversation.py index 33b67977..e23a2c28 100644 --- a/vektra-core/src/vektra_core/conversation.py +++ b/vektra-core/src/vektra_core/conversation.py @@ -125,17 +125,24 @@ async def create_conversation( namespace_id: str, key_id: UUID, title: str | None = None, + conversation_id: UUID | None = None, ) -> UUID: - """Create a new conversation row. Returns the conversation UUID.""" + """Create a new conversation row. Returns the conversation UUID. + + If *conversation_id* is provided, uses it as the primary key instead + of generating one server-side. This supports flows where the caller + has already allocated an ID (e.g. learn endpoint auto-generation). + """ async with self._session_factory() as session: + values: dict[str, Any] = { + "namespace_id": namespace_id, + "key_id": key_id, + "title": title, + } + if conversation_id is not None: + values["id"] = conversation_id stmt = ( - insert(ConversationOrm) - .values( - namespace_id=namespace_id, - key_id=key_id, - title=title, - ) - .returning(ConversationOrm.id) + insert(ConversationOrm).values(**values).returning(ConversationOrm.id) ) result = await session.execute(stmt) conversation_id = result.scalar_one() @@ -147,6 +154,32 @@ async def create_conversation( ) return conversation_id + async def ensure_conversation( + self, + conversation_id: UUID, + namespace_id: str, + key_id: UUID, + ) -> None: + """Create the conversation row if it does not already exist. + + Used when the caller provides a conversation_id (e.g. client-generated + or learn endpoint auto-generated) and the row may or may not exist yet. + """ + async with self._session_factory() as session: + from sqlalchemy.dialects.postgresql import insert as pg_insert + + stmt = ( + pg_insert(ConversationOrm) + .values( + id=conversation_id, + namespace_id=namespace_id, + key_id=key_id, + ) + .on_conflict_do_nothing(index_elements=["id"]) + ) + await session.execute(stmt) + await session.commit() + async def get_history(self, conversation_id: UUID) -> list[dict[str, str | None]]: """Return decrypted conversation history ordered by turn_number. diff --git a/vektra-learn/src/vektra_learn/api.py b/vektra-learn/src/vektra_learn/api.py index 88b8665d..b080b254 100644 --- a/vektra-learn/src/vektra_learn/api.py +++ b/vektra-learn/src/vektra_learn/api.py @@ -501,11 +501,33 @@ async def course_query( if req.conversation_id is None: req = req.model_copy(update={"conversation_id": uuid4()}) + # Ensure conversation row exists for persistent multi-turn (BUG-014). + # The learn endpoint uses JWT auth (no API key), so key_id is a sentinel. + registry = getattr(request.app.state, "registry", None) + if registry is not None: + try: + conv_store = registry.get("conversation_store", "default") + if ( + hasattr(conv_store, "ensure_conversation") + and req.conversation_id is not None + ): + _LEARN_SENTINEL_KEY_ID = UUID("00000000-0000-0000-0000-000000000000") + await conv_store.ensure_conversation( + conversation_id=req.conversation_id, + namespace_id=namespace, + key_id=_LEARN_SENTINEL_KEY_ID, + ) + except ValueError: + pass # conversation store not registered + except Exception as exc: + import structlog + + structlog.get_logger(__name__).warning( + "conversation_create_failed", error=str(exc) + ) + # Build course-scoped query and delegate to pipeline query_req = build_course_query(req, namespace=namespace, course_id=course_id) - - # Get pipeline from ProviderRegistry - registry = getattr(request.app.state, "registry", None) if registry is None: err = ErrorResponse( category=ErrorCategory.TRANSIENT, From 2b82c7034133a3fc0a3125e55cdad78ad13e7502 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:27:50 +0000 Subject: [PATCH 05/33] fix(core): prevent LLM from exposing RAG internals in any language The system prompt told the LLM not to reference sources, but the instruction was in English only and too generic. The model would use translated equivalents like "brani forniti" in Italian, which confuses users who don't know a retrieval system exists. Changes to system.j2: - Explicit blacklist of forbidden expressions in both English and Italian (extensible to other languages) - Reframe assistant persona as "knowledgeable" rather than "with access to a knowledge base" - Provide concrete alternative phrasings for partial-knowledge scenarios - Structure instructions with clear sections for context handling and user communication Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/templates/system.j2 | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index d76f6f7e..6df10f3a 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -1,8 +1,14 @@ -You are a helpful assistant with access to a curated knowledge base. +You are a knowledgeable assistant. Answer questions using the reference material provided in tags. {% if namespace and namespace != "default" %}Namespace: {{ namespace }} {% endif %} -The user's message contains retrieved reference material inside tags. Each element is a passage from the knowledge base, not something the user wrote. Base your answer on these sources. -If the context does not contain enough information to answer the question, say so clearly. -Do not make up information that is not present in the context. -Do not mention or reference the context, sources, passages, or documents to the user. The user is not aware of the retrieval system. Answer naturally as if you know the information directly. -If a source passage is incomplete or cut off, use the available information without commenting on the truncation. +## How to use the context +- Each element is a passage from the knowledge base, not something the user wrote. +- Base your answer strictly on these sources. Do not invent information absent from them. +- If the sources do not cover the question fully, say what you can answer and note that your knowledge on the remaining points is limited. Do NOT explain why (no mention of context, sources, documents, retrieval, chunks, passages, or similar). +- If a source is incomplete or cut off, use what is available without commenting on the truncation. + +## How to talk to the user +- The user does not know that a retrieval system exists. They see you as a knowledgeable assistant. +- NEVER use any of the following expressions or variants in any language: "provided passages", "available passages", "passages I was given", "based on the provided documents", "in the texts provided to me", "from the sources I have", "brani forniti", "testi forniti", "documenti forniti", "fonti a disposizione", "materiale fornito". +- Instead, speak as if the knowledge is yours. Use natural phrases such as "based on the available information", "as far as I know", "the information I have covers..." or their equivalent in the conversation language. +- When you cannot answer fully, prefer: "I don't have enough information on that specific point" over any phrasing that reveals the existence of source documents. From a3676685b82d219e2d5e8db39df581bb1bd7114a Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:40:57 +0000 Subject: [PATCH 06/33] fix(core): tighten system prompt against implicit RAG exposure Previous prompt still allowed the LLM to say "the text I consulted" and offer to "search for more" - both expose the retrieval mechanism or make promises the system cannot keep. Changes: - Blacklist "consulted/reviewed" phrasing in addition to "provided" - Explicitly forbid offering to search, look up, or provide more information later (the system has no such capability) - Instruct to state incomplete knowledge plainly without hinting that more information exists elsewhere Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/src/vektra_core/templates/system.j2 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index 6df10f3a..f72ae077 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -4,11 +4,12 @@ You are a knowledgeable assistant. Answer questions using the reference material ## How to use the context - Each element is a passage from the knowledge base, not something the user wrote. - Base your answer strictly on these sources. Do not invent information absent from them. -- If the sources do not cover the question fully, say what you can answer and note that your knowledge on the remaining points is limited. Do NOT explain why (no mention of context, sources, documents, retrieval, chunks, passages, or similar). +- If the sources do not cover the question fully, answer what you can and state clearly that you do not have information on the remaining points. Do NOT explain why or hint that more information might exist elsewhere. - If a source is incomplete or cut off, use what is available without commenting on the truncation. ## How to talk to the user - The user does not know that a retrieval system exists. They see you as a knowledgeable assistant. -- NEVER use any of the following expressions or variants in any language: "provided passages", "available passages", "passages I was given", "based on the provided documents", "in the texts provided to me", "from the sources I have", "brani forniti", "testi forniti", "documenti forniti", "fonti a disposizione", "materiale fornito". -- Instead, speak as if the knowledge is yours. Use natural phrases such as "based on the available information", "as far as I know", "the information I have covers..." or their equivalent in the conversation language. -- When you cannot answer fully, prefer: "I don't have enough information on that specific point" over any phrasing that reveals the existence of source documents. +- NEVER imply that you consulted, received, or were given any text, document, or material. Do not say "the text I consulted", "the part I reviewed", "the material available to me", or equivalents in any language (e.g. "il testo che ho consultato", "brani forniti", "testi forniti", "documenti forniti", "fonti a disposizione", "materiale fornito", "nella parte che ho consultato"). +- Speak as if the knowledge is yours. Present facts directly without attributing them to an external source. +- When your information is incomplete, say so plainly: "I don't have information on that point." Do not suggest you could search for more, look it up, or provide it later. You cannot do any of those things. +- NEVER offer to "search", "look up", "find", or "provide more information later". Each answer is self-contained. If you don't have it now, you won't have it next time either. From d06dba6df67e6e834b202153fd3ecec13ad53552 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 18:49:17 +0000 Subject: [PATCH 07/33] fix(core): simplify system prompt to improve LLM compliance The previous structured prompt was too verbose for gpt-5-mini, which ignored the blacklist and used "brani forniti" anyway. The word "passage" in the prompt itself was being translated and echoed back. Rewritten as a compact prompt with numbered critical rules. Removed all words the model could recycle (passage, document, source). Added "brani" to the forbidden list since gpt-5-mini translates "passage" to this Italian word. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/vektra_core/templates/system.j2 | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index f72ae077..7921c0ac 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -1,15 +1,10 @@ -You are a knowledgeable assistant. Answer questions using the reference material provided in tags. +You are a knowledgeable assistant. {% if namespace and namespace != "default" %}Namespace: {{ namespace }} {% endif %} -## How to use the context -- Each element is a passage from the knowledge base, not something the user wrote. -- Base your answer strictly on these sources. Do not invent information absent from them. -- If the sources do not cover the question fully, answer what you can and state clearly that you do not have information on the remaining points. Do NOT explain why or hint that more information might exist elsewhere. -- If a source is incomplete or cut off, use what is available without commenting on the truncation. +The user's message includes reference material in tags. Use it to answer. Do not invent facts beyond what is there. If it does not fully cover the question, answer what you can and say you lack information on the rest. -## How to talk to the user -- The user does not know that a retrieval system exists. They see you as a knowledgeable assistant. -- NEVER imply that you consulted, received, or were given any text, document, or material. Do not say "the text I consulted", "the part I reviewed", "the material available to me", or equivalents in any language (e.g. "il testo che ho consultato", "brani forniti", "testi forniti", "documenti forniti", "fonti a disposizione", "materiale fornito", "nella parte che ho consultato"). -- Speak as if the knowledge is yours. Present facts directly without attributing them to an external source. -- When your information is incomplete, say so plainly: "I don't have information on that point." Do not suggest you could search for more, look it up, or provide it later. You cannot do any of those things. -- NEVER offer to "search", "look up", "find", or "provide more information later". Each answer is self-contained. If you don't have it now, you won't have it next time either. +CRITICAL RULES — violating any of these is a failure: +1. The user has no idea that reference material exists. You must sound like you simply know the answer. Never reference, quote, or allude to any external text, material, or source in your reply. +2. Forbidden phrases (any language): "provided text", "given passages", "available documents", "the text I consulted", "brani", "testi forniti", "documenti forniti", "fonti", "materiale". +3. Never offer to search, look up, or provide more later. If you do not have the answer now, you will not have it next time. +4. If a source is cut off, use what is there without mentioning the truncation. From 86bae67bf30213401728535d036458619a7a4fd1 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 20:21:23 +0000 Subject: [PATCH 08/33] fix(core): rewrite system prompt following RAG best practices research Rewritten based on research findings (see vektra-internal 20260324-rag-prompt-best-practices-research.md): - 4 rules only (within compliance threshold for smaller models) - Positive framing ("Sound like you simply know") instead of blacklists - Reintroduce clarification to prevent BUG-012 regression - Add language-matching instruction (rule 4) - Remove forbidden phrase lists (brittle, secondary defense) Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/src/vektra_core/templates/system.j2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index 7921c0ac..22332a26 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -1,10 +1,10 @@ You are a knowledgeable assistant. {% if namespace and namespace != "default" %}Namespace: {{ namespace }} {% endif %} -The user's message includes reference material in tags. Use it to answer. Do not invent facts beyond what is there. If it does not fully cover the question, answer what you can and say you lack information on the rest. +The user's message contains reference material inside tags. Each element is retrieved reference content, not something the user wrote. Use only this material to answer. If it does not fully cover the question, answer what you can and say you do not have information on the rest. -CRITICAL RULES — violating any of these is a failure: -1. The user has no idea that reference material exists. You must sound like you simply know the answer. Never reference, quote, or allude to any external text, material, or source in your reply. -2. Forbidden phrases (any language): "provided text", "given passages", "available documents", "the text I consulted", "brani", "testi forniti", "documenti forniti", "fonti", "materiale". -3. Never offer to search, look up, or provide more later. If you do not have the answer now, you will not have it next time. -4. If a source is cut off, use what is there without mentioning the truncation. +Rules: +1. Sound like you simply know the answer. The user is not aware that reference material exists. Never mention, quote, or allude to texts, documents, sources, passages, or any material you were given. +2. Never offer to search, look up, or provide more information later. You cannot do that. +3. If a source is cut off, use what is available without commenting on the truncation. +4. Respond in the same language the user writes in. From e21214660381b66fea54ffee5ff85ab610215d43 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 21:20:39 +0000 Subject: [PATCH 09/33] fix(core): wire VEKTRA_LLM_API_KEY to litellm and add extra_body support VEKTRA_LLM_API_KEY existed in config but was never passed to litellm, forcing users to set provider-specific env vars (OPENAI_API_KEY) even for OpenAI-compatible endpoints like vLLM. Changes: - LitellmProvider now passes config.api_key to litellm as api_key kwarg - Add VEKTRA_LLM_EXTRA_BODY config field (JSON dict) for provider- specific params (e.g., chat_template_kwargs to disable thinking on Qwen3.5/DeepSeek-R1 via vLLM) - Update configuration.md with vLLM setup instructions and correct VEKTRA_LLM_API_KEY documentation Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/reference/configuration.md | 19 +++++++++++++------ .../vektra_core/providers/litellm_provider.py | 4 ++++ vektra-shared/src/vektra_shared/config.py | 9 +++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d91c99c9..45c4abd2 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -34,14 +34,21 @@ If using a cloud provider, also set the corresponding API key (see [LLM provider ### LLM provider keys -Set one based on your `VEKTRA_LLM_PROVIDER`: +Use `VEKTRA_LLM_API_KEY` for any provider. It is passed directly to litellm and works with OpenAI, Anthropic, vLLM, and any OpenAI-compatible endpoint. -| Variable | Provider | -|----------|----------| -| `OPENAI_API_KEY` | OpenAI (`openai/*`) | -| `ANTHROPIC_API_KEY` | Anthropic (`anthropic/*`) | +Provider-specific environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) are also recognized by litellm as a fallback, but `VEKTRA_LLM_API_KEY` takes precedence when set. -These are standard provider environment variables recognized by litellm. `VEKTRA_LLM_API_KEY` can also be used as a generic alternative. +### Using a local vLLM instance + +```bash +VEKTRA_LLM_PROVIDER=openai//models/your-model-name +VEKTRA_LLM_API_KEY= +VEKTRA_LLM_API_BASE=http://:8000/v1 +# For models with thinking mode (e.g. Qwen3.5, DeepSeek-R1), disable it: +VEKTRA_LLM_EXTRA_BODY={"chat_template_kwargs": {"enable_thinking": false}} +``` + +The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35-27b`). ## Embedding diff --git a/vektra-core/src/vektra_core/providers/litellm_provider.py b/vektra-core/src/vektra_core/providers/litellm_provider.py index 847e50af..b0ca180e 100644 --- a/vektra-core/src/vektra_core/providers/litellm_provider.py +++ b/vektra-core/src/vektra_core/providers/litellm_provider.py @@ -38,8 +38,12 @@ class LitellmProvider: def __init__(self, config: LLMConfig) -> None: self._config = config self._base_kwargs: dict[str, Any] = {} + if config.api_key: + self._base_kwargs["api_key"] = config.api_key if config.api_base: self._base_kwargs["api_base"] = config.api_base + if config.extra_body: + self._base_kwargs["extra_body"] = config.extra_body @property def model_name(self) -> str: diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index beb9f32c..02064864 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -9,6 +9,8 @@ from __future__ import annotations +from typing import Any + from pydantic import Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -31,6 +33,11 @@ class LLMConfig(BaseSettings): alias="VEKTRA_LLM_API_BASE", description="Custom API base URL for OpenAI-compatible providers (e.g., vLLM).", ) + extra_body: dict[str, Any] | None = Field( + None, + alias="VEKTRA_LLM_EXTRA_BODY", + description="Extra JSON body params passed to the LLM API (e.g., chat_template_kwargs for vLLM).", + ) fallback_model: str | None = Field( None, alias="VEKTRA_LLM_FALLBACK_MODEL", @@ -415,6 +422,7 @@ class VektraSettings(BaseSettings): ) llm_api_key: str | None = Field(None, alias="VEKTRA_LLM_API_KEY") llm_api_base: str | None = Field(None, alias="VEKTRA_LLM_API_BASE") + llm_extra_body: dict[str, Any] | None = Field(None, alias="VEKTRA_LLM_EXTRA_BODY") llm_fallback_model: str | None = Field(None, alias="VEKTRA_LLM_FALLBACK_MODEL") llm_fallback_timeout_ms: int = Field(60000, alias="VEKTRA_LLM_FALLBACK_TIMEOUT_MS") llm_context_only_enabled: bool = Field( @@ -523,6 +531,7 @@ def as_llm_config(self) -> LLMConfig: "VEKTRA_LLM_PROVIDER": self.llm_provider, "VEKTRA_LLM_API_KEY": self.llm_api_key, "VEKTRA_LLM_API_BASE": self.llm_api_base, + "VEKTRA_LLM_EXTRA_BODY": self.llm_extra_body, "VEKTRA_LLM_FALLBACK_MODEL": self.llm_fallback_model, "VEKTRA_LLM_FALLBACK_TIMEOUT_MS": self.llm_fallback_timeout_ms, "VEKTRA_LLM_CONTEXT_ONLY_ENABLED": self.llm_context_only_enabled, From a34a551d542a123e40a2673b79b56e3b8e03da1d Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 21:28:41 +0000 Subject: [PATCH 10/33] refactor(learn): clean up conversation persistence code - Move structlog import to top level (was inline in except block) - Extract _LEARN_SENTINEL_KEY_ID as module-level constant - Fix import ordering (ruff I001) Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-learn/src/vektra_learn/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vektra-learn/src/vektra_learn/api.py b/vektra-learn/src/vektra_learn/api.py index b080b254..2f85c678 100644 --- a/vektra-learn/src/vektra_learn/api.py +++ b/vektra-learn/src/vektra_learn/api.py @@ -16,6 +16,7 @@ import httpx import jwt +import structlog from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import StreamingResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer @@ -47,6 +48,9 @@ http_status_for, ) +# Sentinel key_id for learn-originated conversations (JWT auth has no API key). +_LEARN_SENTINEL_KEY_ID = UUID("00000000-0000-0000-0000-000000000000") + router = APIRouter(prefix="/api/v1/learn", tags=["learn"]) @@ -511,7 +515,6 @@ async def course_query( hasattr(conv_store, "ensure_conversation") and req.conversation_id is not None ): - _LEARN_SENTINEL_KEY_ID = UUID("00000000-0000-0000-0000-000000000000") await conv_store.ensure_conversation( conversation_id=req.conversation_id, namespace_id=namespace, @@ -520,8 +523,6 @@ async def course_query( except ValueError: pass # conversation store not registered except Exception as exc: - import structlog - structlog.get_logger(__name__).warning( "conversation_create_failed", error=str(exc) ) From d717714a2ecec8dbb13dcc41ccc57b884d4edeec Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 22:01:44 +0000 Subject: [PATCH 11/33] chore(backlog): add BUG-015, BUG-016, TECH-002, DEBT-010 for retrieval quality BUG-015: reranker scores discarded (threshold applied to wrong scores) BUG-016: English-only reranker on Italian content TECH-002: RAG evaluation harness (50-question dataset, RAGAS) DEBT-010: recalibrate relevance threshold with empirical data All traced to analysis in vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 9b1b2545..4fba8e42 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -107,6 +107,110 @@ --- +### BUG-015: Reranker scores discarded after reranking — threshold applied to wrong scores + +**Status**: planned | **Priority**: critical | **Created**: 2026-03-24 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` + +**Context**: `RerankerService.rerank()` (reranker.py:54-60) reorders results but returns the original `SearchResult` objects with their cosine similarity scores intact. The flashrank/cross-encoder scores are used only for ordering, then discarded. The `_apply_retrieval_filter` (pipeline.py:100) then applies `VEKTRA_MIN_RELEVANCE_SCORE=0.3` to these original cosine scores, not the reranker scores. The reranker's relevance judgment and the threshold filter are effectively disconnected: a chunk the reranker ranks highly can still be filtered out if its original cosine similarity was below 0.3. + +**Root cause**: The reranker implementation (commit dcb0b54, 2026-03-05) was designed to only reorder, not to propagate scores. The test `test_rerank_returns_top_k_in_order` verifies ordering but not score propagation. ADR-0014 and the implementation plan do not specify score handling. + +**Traceability**: ARCH-056, ADR-0021, ADR-0014 + +**Acceptance criteria**: +- [ ] `RerankerService.rerank()` propagates reranker scores to `SearchResult.score` (or a new field) +- [ ] `_apply_retrieval_filter` uses the correct score (reranker if available, cosine if not) +- [ ] Score normalization: all reranker outputs normalized to 0-1 at the reranker boundary +- [ ] When reranker is disabled, behavior unchanged (cosine scores, same threshold) +- [ ] Test verifies score values after reranking, not just ordering +- [ ] Config `VEKTRA_MIN_RELEVANCE_SCORE` description updated to reflect it applies to the active scoring stage + +--- + +### BUG-016: English-only reranker produces random scores on Italian content + +**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` +**Depends on**: BUG-015 (score propagation must work before reranker swap is meaningful) + +**Context**: The default reranker model `ms-marco-MiniLM-L-12-v2` (via flashrank) is trained exclusively on English MS MARCO data. Its English-uncased tokenizer splits Italian words into meaningless subword fragments. On Italian text, the reranker produces essentially random relevance scores, potentially degrading retrieval by reordering correctly-retrieved chunks into a worse order. + +The choice was made during the hybrid search design phase (2026-02-07) optimizing for deployment constraints (4MB, no GPU, 50ms). Multilingual support was delegated entirely to the embedding model. The RAG tuning campaign (600 queries, 6 combos) held the reranker constant and never evaluated alternatives. + +The system must support both Italian and English content/queries (and mixed), so the solution must be multilingual, not Italian-specific. + +**Alternatives evaluated** (see analysis doc for full comparison): + +| Model | Params | Multilingual | Quality | Memory | Config change only? | +|-------|--------|-------------|---------|--------|---------------------| +| bge-reranker-v2-m3 | 568M | 100+ langs, best Mr.TyDi | High | ~1.2GB GPU | Yes | +| jina-reranker-v2-base-multilingual | 278M | 100+ langs | Good | ~600MB GPU | Yes | +| ms-marco-MultiBERT-L-12 (flashrank) | ~150M | 100+ langs | Terrible (26.91 NDCG) | ~150MB CPU | Yes, but do not use | + +The `rerankers` library already supports cross-encoder backends. No code changes needed: +``` +VEKTRA_RERANK_PROVIDER=cross-encoder +VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 +``` + +**Traceability**: ARCH-036, ADR-0021 + +**Acceptance criteria**: +- [ ] Default reranker model changed to a multilingual model that supports Italian and English +- [ ] English-only model remains available via config for English-only deployments +- [ ] Performance validated on Italian and English test queries (requires eval harness, TECH-002) +- [ ] Documentation updated (configuration.md, .env.example) with multilingual model guidance +- [ ] Memory and latency impact documented + +--- + +### TECH-002: RAG evaluation harness + +**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` + +**Context**: The RAG tuning campaign (2026-03-14-18) used manual testing across 600 queries with qualitative metrics. There is no automated, reproducible way to evaluate retrieval quality when components change (embedding model, reranker, threshold, chunk size). This gap allowed BUG-015 and BUG-016 to go undetected: component interactions were never tested systematically. + +**Scope**: +- 50-question curated dataset (Italian Constitution + at least one English-language source) +- Two-stage evaluation: retrieval-only (fast, no LLM) and end-to-end (RAGAS metrics) +- `make eval-retrieval` and `make eval-e2e` targets +- Paired comparison support (same questions, two configs) +- JSONL results storage for historical tracking + +**Metrics**: context recall (primary), context precision, faithfulness, answer relevancy. + +**Traceability**: ARCH-050 (three-tier evaluation strategy), ADR-0019 + +**Acceptance criteria**: +- [ ] Curated test dataset with ground truth contexts (JSON, versioned in repo) +- [ ] `make eval-retrieval` runs retrieval-only evaluation and outputs metrics +- [ ] `make eval-e2e` runs full pipeline evaluation with RAGAS +- [ ] Baseline results recorded for current Combo D configuration +- [ ] Documentation on how to add test questions and run evaluations + +--- + +### DEBT-010: Recalibrate relevance threshold with empirical data + +**Status**: planned | **Priority**: medium | **Created**: 2026-03-24 +**Depends on**: BUG-015, BUG-016, TECH-002 + +**Context**: `VEKTRA_MIN_RELEVANCE_SCORE=0.3` was set per ADR-0021 for cosine similarity with `all-MiniLM-L6-v2` (Phase 1 embedding model). It was never varied in the tuning campaign and not recalibrated when: (a) the embedding model changed to `paraphrase-multilingual-MiniLM-L12-v2`, (b) hybrid search with RRF was enabled, (c) the reranker was added. Different scoring stages produce different distributions (cosine 0.2-0.8 cluster, flashrank bimodal near 0/1, RRF reciprocal). A single threshold cannot serve all correctly. + +Literature consensus: use top-k as primary control, low absolute threshold (0.15-0.2) as safety net. Consider hybrid filtering (absolute minimum + relative percentile). + +**Traceability**: ARCH-056, ADR-0021 + +**Acceptance criteria**: +- [ ] Threshold tested at 0.1, 0.15, 0.2, 0.25, 0.3 using eval harness (TECH-002) +- [ ] Optimal threshold determined for the active reranker + embedding model combination +- [ ] ADR-0021 updated with new calibration data +- [ ] Configuration supports different thresholds for reranked vs non-reranked modes (or hybrid filter) + +--- + ### DEBT-009: Debug logging for rewritten queries **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 From 2fbec583c9ecc6ab2528940558c421a724aa8f7e Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 22:25:45 +0000 Subject: [PATCH 12/33] fix(build): harden supply chain - eliminate uv pip install outside lockfile Pin litellm <1.82.7 (versions 1.82.7-1.82.8 contain credential stealer). Replace direct uv pip install of qdrant-client/fastembed with lockfile-based --extra sparse --extra qdrant. Drop unused torchvision. All dependencies now verified against lockfile hashes via uv sync --frozen. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 6 ++---- uv.lock | 2 +- vektra-core/pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3351715..251f7bea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,11 +70,9 @@ COPY vektra-app/src vektra-app/src # Include optional extras for Phase 2 vector store and sparse search support. # When INSTALL_UNSTRUCTURED=true, also install the Unstructured PDF extractor. ARG INSTALL_UNSTRUCTURED=false -RUN uv sync --frozen --no-editable --no-dev \ - && uv pip install 'qdrant-client==1.17.0' 'fastembed==0.7.4' \ +RUN uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant \ && if [ "$INSTALL_UNSTRUCTURED" = "true" ]; then \ - uv pip install 'torch' 'torchvision' --index-url https://download.pytorch.org/whl/cpu --reinstall \ - && uv pip install 'unstructured[pdf]>=0.15' 'pi-heif' 'sentence-transformers' --reinstall; \ + uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant --extra ocr; \ fi # -------------------------------------------------------------------------- diff --git a/uv.lock b/uv.lock index 93a31c07..7928b3a8 100644 --- a/uv.lock +++ b/uv.lock @@ -5113,7 +5113,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.115" }, { name = "jinja2", specifier = ">=3.1" }, - { name = "litellm", specifier = ">=1.40" }, + { name = "litellm", specifier = ">=1.40,<1.82.7" }, { name = "presidio-analyzer", specifier = ">=2.2" }, { name = "presidio-anonymizer", specifier = ">=2.2" }, { name = "pydantic", specifier = ">=2,<3" }, diff --git a/vektra-core/pyproject.toml b/vektra-core/pyproject.toml index fcf16d2b..a14a1d12 100644 --- a/vektra-core/pyproject.toml +++ b/vektra-core/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "vektra-shared", "fastapi>=0.115", "pydantic>=2,<3", - "litellm>=1.40", + "litellm>=1.40,<1.82.7", "jinja2>=3.1", "structlog>=24.0", "presidio-analyzer>=2.2", From 838b40ecb91607eb71cf8f972aab00a5ee2b575d Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 22:49:12 +0000 Subject: [PATCH 13/33] fix(core): propagate reranker scores to SearchResult (BUG-015) RerankerService.rerank() was discarding reranker scores and returning original cosine similarities, making the relevance threshold filter operate on the wrong values. Now reranker scores replace SearchResult.score with sigmoid normalization for cross-encoder logits. Original score preserved in SearchResult.original_score for debugging. Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/src/vektra_core/reranker.py | 35 ++++++- vektra-core/tests/test_reranker.py | 113 ++++++++++++++++++++++- vektra-shared/src/vektra_shared/types.py | 1 + 3 files changed, 142 insertions(+), 7 deletions(-) diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 94bea399..12b48112 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -2,11 +2,18 @@ Runs cross-encoder or flashrank reranking on vector search results. CPU-bound inference is offloaded to a thread via asyncio.to_thread(). + +Score propagation (BUG-015): reranker scores replace the original vector +similarity scores on SearchResult.score. The original score is preserved +in SearchResult.original_score for debugging. Scores are normalized to +[0, 1] via sigmoid when raw logits are detected (cross-encoder providers). """ from __future__ import annotations import asyncio +import dataclasses +import math import structlog @@ -23,6 +30,15 @@ } +def _sigmoid(x: float) -> float: + """Numerically stable sigmoid for cross-encoder logits.""" + if x >= 0: + z = math.exp(-x) + return 1.0 / (1.0 + z) + z = math.exp(x) + return z / (1.0 + z) + + class RerankerService: """Wraps the rerankers library for scoring and reordering search results.""" @@ -51,11 +67,24 @@ async def rerank( docs=docs, ) - # Build a mapping from original doc index to SearchResult + # Extract scores and detect whether normalization is needed. + # FlashRank produces sigmoid scores in [0, 1]; cross-encoder + # produces raw logits that can be negative or > 1. + top_items = ranked.results[:top_k] + raw_scores = [float(item.score) for item in top_items] + needs_sigmoid = any(s < 0.0 or s > 1.0 for s in raw_scores) + reranked: list[SearchResult] = [] - for item in ranked.results[:top_k]: + for item, raw_score in zip(top_items, raw_scores): original = results[item.doc_id] - reranked.append(original) + normalized = _sigmoid(raw_score) if needs_sigmoid else raw_score + reranked.append( + dataclasses.replace( + original, + score=normalized, + original_score=original.score, + ) + ) return reranked diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index 742e221c..2903b848 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -7,6 +7,7 @@ from vektra_core.reranker import ( RerankerService, _default_model_for_provider, + _sigmoid, create_reranker, ) from vektra_shared.config import RerankConfig @@ -40,11 +41,11 @@ async def test_rerank_returns_top_k_in_order(): _make_result(0.9, "doc C"), ] - # Simulate rerankers output: list of items with doc_id attribute + # FlashRank-style scores (already 0-1) ranked_items = [ - SimpleNamespace(doc_id=2), # doc C first - SimpleNamespace(doc_id=0), # doc A second - SimpleNamespace(doc_id=1), # doc B third + SimpleNamespace(doc_id=2, score=0.95), # doc C first + SimpleNamespace(doc_id=0, score=0.80), # doc A second + SimpleNamespace(doc_id=1, score=0.10), # doc B third ] mock_ranker = MagicMock() mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) @@ -60,6 +61,87 @@ async def test_rerank_returns_top_k_in_order(): ) +async def test_rerank_propagates_flashrank_scores(): + """FlashRank scores (0-1) are propagated as-is (BUG-015).""" + results = [ + _make_result(0.5, "doc A"), + _make_result(0.3, "doc B"), + ] + + ranked_items = [ + SimpleNamespace(doc_id=1, score=0.85), + SimpleNamespace(doc_id=0, score=0.40), + ] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("query", results, top_k=2) + + # Reranker scores replace vector scores + assert reranked[0].score == 0.85 + assert reranked[1].score == 0.40 + # Original vector scores preserved + assert reranked[0].original_score == 0.3 + assert reranked[1].original_score == 0.5 + + +async def test_rerank_normalizes_cross_encoder_logits(): + """Cross-encoder logits (can be negative / > 1) are sigmoid-normalized (BUG-015).""" + results = [ + _make_result(0.6, "doc A"), + _make_result(0.4, "doc B"), + ] + + ranked_items = [ + SimpleNamespace(doc_id=0, score=3.5), # positive logit + SimpleNamespace(doc_id=1, score=-2.0), # negative logit + ] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("query", results, top_k=2) + + # Scores normalized via sigmoid + assert reranked[0].score == _sigmoid(3.5) + assert reranked[1].score == _sigmoid(-2.0) + # Normalized scores are in (0, 1) + assert 0.0 < reranked[0].score < 1.0 + assert 0.0 < reranked[1].score < 1.0 + # Original scores preserved + assert reranked[0].original_score == 0.6 + assert reranked[1].original_score == 0.4 + + +async def test_rerank_preserves_metadata(): + """Reranked results keep all original fields except score.""" + doc_id = uuid4() + results = [ + SearchResult( + chunk_id="chunk-1", + score=0.7, + text_snippet="hello", + document_id=doc_id, + document_version=3, + metadata={"page": 5}, + ), + ] + ranked_items = [SimpleNamespace(doc_id=0, score=0.99)] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("q", results, top_k=1) + + assert reranked[0].chunk_id == "chunk-1" + assert reranked[0].document_id == doc_id + assert reranked[0].document_version == 3 + assert reranked[0].metadata == {"page": 5} + assert reranked[0].score == 0.99 + assert reranked[0].original_score == 0.7 + + # --------------------------------------------------------------------------- # create_reranker factory # --------------------------------------------------------------------------- @@ -95,3 +177,26 @@ def test_default_model_cross_encoder(): def test_default_model_unknown_provider(): assert _default_model_for_provider("custom") == "custom" + + +# --------------------------------------------------------------------------- +# _sigmoid +# --------------------------------------------------------------------------- + + +def test_sigmoid_zero(): + assert _sigmoid(0.0) == 0.5 + + +def test_sigmoid_large_positive(): + assert _sigmoid(10.0) > 0.999 + + +def test_sigmoid_large_negative(): + assert _sigmoid(-10.0) < 0.001 + + +def test_sigmoid_extreme_values_no_overflow(): + """Numerically stable sigmoid must not overflow on extreme logits.""" + assert _sigmoid(1000.0) == 1.0 + assert _sigmoid(-1000.0) == 0.0 diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index 797d65e3..0c056fea 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -156,6 +156,7 @@ class SearchResult: document_id: UUID document_version: int = 1 # from SourceDocument.version (REQ-056) metadata: dict[str, Any] = field(default_factory=dict) + original_score: float | None = None # pre-reranker score (BUG-015) @dataclass From 861eae8d37683859c58380a7861d9dfba08aa692 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 22:55:07 +0000 Subject: [PATCH 14/33] feat(core): switch to multilingual reranker and lower threshold (BUG-016, DEBT-010) Default reranker changed from flashrank/ms-marco-MiniLM-L-12-v2 (English-only) to cross-encoder/BAAI/bge-reranker-v2-m3 (multilingual). Threshold lowered from 0.3 to 0.15 as interim safety net - top-k is the primary chunk count control. Existing flashrank config remains supported for English-only lightweight deployments. Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 6 +++--- docs/reference/configuration.md | 6 +++--- vektra-core/src/vektra_core/reranker.py | 2 +- vektra-core/tests/test_reranker.py | 5 +---- vektra-shared/src/vektra_shared/config.py | 10 +++++----- vektra-shared/tests/test_config.py | 6 +++--- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 67e08912..d5541929 100644 --- a/.env.example +++ b/.env.example @@ -77,7 +77,7 @@ # -------------------------------------------------------------------------- # VEKTRA_QUERY_PIPELINE=advanced -# VEKTRA_MIN_RELEVANCE_SCORE=0.3 +# VEKTRA_MIN_RELEVANCE_SCORE=0.15 # VEKTRA_CHUNK_DEDUP_ENABLED=true # VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 @@ -89,8 +89,8 @@ # Reranking # VEKTRA_RERANK_ENABLED=true -# VEKTRA_RERANK_PROVIDER=flashrank -# VEKTRA_RERANK_MODEL= +# VEKTRA_RERANK_PROVIDER=cross-encoder +# VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 # VEKTRA_RERANK_TOP_K=5 # -------------------------------------------------------------------------- diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 45c4abd2..ac6802a5 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -74,7 +74,7 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_QUERY_PIPELINE` | str | `advanced` | Pipeline implementation: `simple`, `advanced` | -| `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.3` | Minimum cosine similarity for chunk inclusion (0.0-1.0). Chunks below this threshold are filtered out. | +| `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.15` | Minimum relevance score for chunk inclusion (0.0-1.0). Safety net filter; top-k is the primary control. | | `VEKTRA_CHUNK_DEDUP_ENABLED` | bool | `true` | Deduplicate overlapping adjacent chunks from the same document | | `VEKTRA_RESPONSE_TOKEN_RESERVE` | int | `2048` | Tokens reserved for LLM response generation | | `VEKTRA_CONTEXT_CHUNK_RATIO` | float | `0.6` | Fraction of context window allocated to retrieved chunks (0.0-1.0) | @@ -92,8 +92,8 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_RERANK_ENABLED` | bool | `true` | Enable cross-encoder reranking after retrieval | -| `VEKTRA_RERANK_PROVIDER` | str | `flashrank` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | -| `VEKTRA_RERANK_MODEL` | str | - | Provider-specific reranking model name | +| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | +| `VEKTRA_RERANK_MODEL` | str | `BAAI/bge-reranker-v2-m3` | Multilingual reranking model. For English-only lightweight deployments: provider=`flashrank`, model=`ms-marco-MiniLM-L-12-v2` | | `VEKTRA_RERANK_TOP_K` | int | `5` | Final top-k results after reranking | ## Ingestion diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 12b48112..9d7e9c70 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -125,6 +125,6 @@ def _default_model_for_provider(provider: str) -> str: """Return a sensible default model for each provider.""" defaults = { "flashrank": "ms-marco-MiniLM-L-12-v2", - "cross-encoder": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "cross-encoder": "BAAI/bge-reranker-v2-m3", } return defaults.get(provider, provider) diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index 2903b848..55072b54 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -169,10 +169,7 @@ def test_default_model_flashrank(): def test_default_model_cross_encoder(): - assert ( - _default_model_for_provider("cross-encoder") - == "cross-encoder/ms-marco-MiniLM-L-6-v2" - ) + assert _default_model_for_provider("cross-encoder") == "BAAI/bge-reranker-v2-m3" def test_default_model_unknown_provider(): diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 02064864..214a49d3 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -150,12 +150,12 @@ class RerankConfig(BaseSettings): description="Enable reranking in AdvancedQueryPipeline.", ) provider: str = Field( - "flashrank", + "cross-encoder", alias="VEKTRA_RERANK_PROVIDER", description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere'.", ) model: str | None = Field( - None, + "BAAI/bge-reranker-v2-m3", alias="VEKTRA_RERANK_MODEL", description="Provider-specific reranking model name.", ) @@ -180,9 +180,9 @@ class QueryPipelineConfig(BaseSettings): description="QueryPipeline implementation: 'simple' (Phase 1), 'advanced' (Phase 2).", ) min_relevance_score: float = Field( - 0.3, + 0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE", - description="Minimum cosine similarity for chunk inclusion (ARCH-056).", + description="Minimum relevance score for chunk inclusion (ARCH-056). Safety net filter; top-k is the primary control.", ) chunk_dedup_enabled: bool = Field( True, @@ -452,7 +452,7 @@ class VektraSettings(BaseSettings): # Query pipeline query_pipeline: str = Field("advanced", alias="VEKTRA_QUERY_PIPELINE") - min_relevance_score: float = Field(0.3, alias="VEKTRA_MIN_RELEVANCE_SCORE") + min_relevance_score: float = Field(0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE") chunk_dedup_enabled: bool = Field(True, alias="VEKTRA_CHUNK_DEDUP_ENABLED") response_token_reserve: int = Field(2048, alias="VEKTRA_RESPONSE_TOKEN_RESERVE") context_chunk_ratio: float = Field(0.6, alias="VEKTRA_CONTEXT_CHUNK_RATIO") diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 15095e14..329e6f50 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -76,8 +76,8 @@ class TestRerankConfig: def test_defaults(self) -> None: cfg = RerankConfig() assert cfg.enabled is True - assert cfg.provider == "flashrank" - assert cfg.model is None + assert cfg.provider == "cross-encoder" + assert cfg.model == "BAAI/bge-reranker-v2-m3" assert cfg.top_k == 5 def test_env_var_override(self) -> None: @@ -147,7 +147,7 @@ def test_rewrite_and_rerank_defaults(self) -> None: assert cfg.rewrite.enabled is True assert cfg.rewrite.model is None assert cfg.rerank.enabled is True - assert cfg.rerank.provider == "flashrank" + assert cfg.rerank.provider == "cross-encoder" assert cfg.rerank.top_k == 5 From 3484d004ad8f281fbddaa62eacc80b9cf4b2a374 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 23:09:48 +0000 Subject: [PATCH 15/33] feat(eval): add retrieval and e2e evaluation harness (TECH-002) Add eval scripts that call the Vektra API and compute retrieval quality metrics (hit rate, MRR, precision@k) and e2e metrics (answer rate, latency percentiles). Includes seed dataset with 5 questions, Makefile targets (eval-retrieval, eval-e2e), and JSONL results output. Full 50-question dataset to be curated on Kalypso with live content. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 3 + Makefile | 8 +- scripts/eval_e2e.py | 238 ++++++++++++++++++++++++++++ scripts/eval_retrieval.py | 317 ++++++++++++++++++++++++++++++++++++++ tests/eval/dataset.jsonl | 5 + 5 files changed, 570 insertions(+), 1 deletion(-) create mode 100644 scripts/eval_e2e.py create mode 100644 scripts/eval_retrieval.py create mode 100644 tests/eval/dataset.jsonl diff --git a/.gitignore b/.gitignore index 6fdd6674..5c43335e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,9 @@ vektra-learn/static/ coverage/ .nyc_output/ +# Evaluation results (generated by eval harness) +tests/eval/results_*.jsonl + # Temporary files tmp/ temp/ diff --git a/Makefile b/Makefile index 78768caf..7f0448d8 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ endif # Targets # -------------------------------------------------------------------------- -.PHONY: help up down health ingest query demo logs test lint reindex batch-ingest +.PHONY: help up down health ingest query demo logs test lint reindex batch-ingest eval-retrieval eval-e2e help: ## Show available targets @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*##/ { \ @@ -98,3 +98,9 @@ reindex: ## Create reindex job (skeleton): make reindex VER=2 [NS=default] batch-ingest: ## Batch ingest files: make batch-ingest DIR=path/to/docs [NS=default] $(if $(DIR),,$(error DIR is required. Usage: make batch-ingest DIR=path/to/docs)) @scripts/batch-ingest.sh "$(DIR)" "$(or $(NS),default)" + +eval-retrieval: ## Run retrieval-only evaluation (no LLM calls) + uv run python scripts/eval_retrieval.py $(EVAL_ARGS) + +eval-e2e: ## Run end-to-end RAG evaluation (with LLM calls) + uv run python scripts/eval_e2e.py $(EVAL_ARGS) diff --git a/scripts/eval_e2e.py b/scripts/eval_e2e.py new file mode 100644 index 00000000..a818fa36 --- /dev/null +++ b/scripts/eval_e2e.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""End-to-end RAG evaluation harness (TECH-002). + +Reads a JSONL dataset, calls the Vektra query API for each question, +and computes answer quality metrics. Optionally uses RAGAS for +faithfulness and answer relevancy scoring. + +Usage: + python scripts/eval_e2e.py [OPTIONS] + +Requires VEKTRA_API_URL and VEKTRA_API_KEY environment variables. +For RAGAS metrics, install: uv pip install ragas datasets +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +DEFAULT_DATASET = "tests/eval/dataset.jsonl" +DEFAULT_OUTPUT = "tests/eval/results_e2e.jsonl" + + +@dataclass +class E2EResult: + id: str + question: str + category: str + language: str + answer: str | None + no_relevant_context: bool + num_sources: int + source_scores: list[float] = field(default_factory=list) + duration_ms: float = 0.0 + error: str | None = None + + +def load_dataset(path: str) -> list[dict]: + entries = [] + with open(path) as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError as e: + print(f"Warning: skipping line {line_num}: {e}", file=sys.stderr) + return entries + + +def evaluate_question(client: httpx.Client, entry: dict, top_k: int) -> E2EResult: + """Run a single RAG query and collect the response.""" + question_id = entry["id"] + question = entry["question"] + namespace = entry.get("namespace", "default") + category = entry.get("category", "unknown") + language = entry.get("language", "unknown") + + t0 = time.monotonic() + try: + resp = client.post( + "/api/v1/query", + json={ + "question": question, + "namespace": namespace, + "top_k": top_k, + }, + ) + resp.raise_for_status() + data = resp.json() + except Exception as e: + return E2EResult( + id=question_id, + question=question, + category=category, + language=language, + answer=None, + no_relevant_context=True, + num_sources=0, + error=str(e), + ) + + duration_ms = (time.monotonic() - t0) * 1000 + sources = data.get("sources", []) + + return E2EResult( + id=question_id, + question=question, + category=category, + language=language, + answer=data.get("answer"), + no_relevant_context=data.get("no_relevant_context", False), + num_sources=len(sources), + source_scores=[s["score"] for s in sources], + duration_ms=duration_ms, + ) + + +def print_summary(results: list[E2EResult], elapsed_s: float) -> None: + total = len(results) + errors = sum(1 for r in results if r.error) + evaluated = total - errors + + if evaluated == 0: + print("No questions evaluated successfully.") + return + + valid = [r for r in results if r.error is None] + answered = sum(1 for r in valid if r.answer is not None) + no_context = sum(1 for r in valid if r.no_relevant_context) + avg_sources = sum(r.num_sources for r in valid) / evaluated + + durations = sorted(r.duration_ms for r in valid) + p50 = durations[len(durations) // 2] + p95 = durations[int(len(durations) * 0.95)] + + print(f"\n{'=' * 60}") + print("End-to-end evaluation results") + print(f"{'=' * 60}") + print(f"Questions: {total} ({errors} errors)") + print(f"Duration: {elapsed_s:.1f}s total") + print("") + print(f"Answered: {answered}/{evaluated} ({answered / evaluated:.0%})") + print(f"No context: {no_context}/{evaluated}") + print(f"Avg sources: {avg_sources:.1f}") + print(f"Latency p50: {p50:.0f}ms") + print(f"Latency p95: {p95:.0f}ms") + + # Breakdown by category + categories = sorted(set(r.category for r in valid)) + if len(categories) > 1: + print("\nBy category:") + for cat in categories: + cat_results = [r for r in valid if r.category == cat] + cat_answered = sum(1 for r in cat_results if r.answer is not None) + print( + f" {cat:<15} answered={cat_answered}/{len(cat_results)} n={len(cat_results)}" + ) + + print(f"{'=' * 60}") + print("\nNote: RAGAS metrics (faithfulness, answer relevancy) require") + print("ground_truth_answer in the dataset and RAGAS installed.") + print("Install: uv pip install ragas datasets") + + +def save_results(results: list[E2EResult], output_path: str) -> None: + with open(output_path, "w") as f: + for r in results: + row = { + "id": r.id, + "question": r.question, + "category": r.category, + "language": r.language, + "answer": r.answer, + "no_relevant_context": r.no_relevant_context, + "num_sources": r.num_sources, + "source_scores": r.source_scores, + "duration_ms": round(r.duration_ms, 1), + } + if r.error: + row["error"] = r.error + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Vektra end-to-end evaluation harness") + parser.add_argument( + "--dataset", + default=DEFAULT_DATASET, + help=f"Path to JSONL dataset (default: {DEFAULT_DATASET})", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help=f"Path for JSONL results output (default: {DEFAULT_OUTPUT})", + ) + parser.add_argument( + "--top-k", type=int, default=5, help="Top-k for query (default: 5)" + ) + args = parser.parse_args() + + api_url = os.environ.get("VEKTRA_API_URL") + api_key = os.environ.get("VEKTRA_API_KEY") + if not api_key: + api_key_file = Path(".api-key") + if api_key_file.exists(): + api_key = api_key_file.read_text().strip() + + if not api_url or not api_key: + print( + "Error: VEKTRA_API_URL and VEKTRA_API_KEY must be set.", + file=sys.stderr, + ) + sys.exit(1) + + dataset = load_dataset(args.dataset) + if not dataset: + print(f"Error: no entries in {args.dataset}", file=sys.stderr) + sys.exit(1) + + print(f"Loaded {len(dataset)} questions from {args.dataset}") + print(f"API: {api_url} top_k={args.top_k}") + + client = httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=120.0, # LLM calls can be slow + ) + + results: list[E2EResult] = [] + t0 = time.monotonic() + for i, entry in enumerate(dataset): + result = evaluate_question(client, entry, args.top_k) + results.append(result) + status = "OK" if result.answer else ("ERR" if result.error else "NO_CTX") + answer_preview = (result.answer or "")[:60].replace("\n", " ") + print( + f" [{i + 1}/{len(dataset)}] {entry['id']} {status} {result.duration_ms:.0f}ms {answer_preview}" + ) + + elapsed = time.monotonic() - t0 + client.close() + + print_summary(results, elapsed) + save_results(results, args.output) + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py new file mode 100644 index 00000000..dadc5c58 --- /dev/null +++ b/scripts/eval_retrieval.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Retrieval-only evaluation harness (TECH-002). + +Reads a JSONL dataset, calls the Vektra search API for each question, +and computes retrieval quality metrics. No LLM calls are made. + +Usage: + python scripts/eval_retrieval.py [OPTIONS] + +Requires VEKTRA_API_URL and VEKTRA_API_KEY environment variables. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +DEFAULT_DATASET = "tests/eval/dataset.jsonl" +DEFAULT_OUTPUT = "tests/eval/results_retrieval.jsonl" + + +@dataclass +class QuestionResult: + id: str + question: str + category: str + language: str + hit: bool # at least one chunk matches expected keywords + reciprocal_rank: float # 1/rank of first relevant chunk (0 if no hit) + precision_at_k: float # relevant chunks / total chunks + num_retrieved: int + num_relevant: int + scores: list[float] = field(default_factory=list) + error: str | None = None + + +def load_dataset(path: str) -> list[dict]: + entries = [] + with open(path) as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError as e: + print(f"Warning: skipping line {line_num}: {e}", file=sys.stderr) + return entries + + +def chunk_matches_keywords(text: str, keywords: list[str]) -> bool: + """Check if a chunk's text contains any of the expected keywords (case-insensitive).""" + text_lower = text.lower() + return any(kw.lower() in text_lower for kw in keywords) + + +def evaluate_question( + client: httpx.Client, + entry: dict, + top_k: int, + search_mode: str, +) -> QuestionResult: + """Run a single search query and compute retrieval metrics.""" + question_id = entry["id"] + question = entry["question"] + keywords = entry.get("expected_keywords", []) + namespace = entry.get("namespace", "default") + category = entry.get("category", "unknown") + language = entry.get("language", "unknown") + + try: + resp = client.post( + "/api/v1/search", + json={ + "query": question, + "namespace": namespace, + "top_k": top_k, + "search_mode": search_mode, + }, + ) + resp.raise_for_status() + data = resp.json() + except Exception as e: + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=False, + reciprocal_rank=0.0, + precision_at_k=0.0, + num_retrieved=0, + num_relevant=0, + error=str(e), + ) + + results = data.get("results", []) + scores = [r["score"] for r in results] + + if not keywords: + # No ground truth: can only report retrieval count and scores + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=False, + reciprocal_rank=0.0, + precision_at_k=0.0, + num_retrieved=len(results), + num_relevant=0, + scores=scores, + ) + + # Compute which chunks are relevant + relevant_mask = [ + chunk_matches_keywords(r.get("text_snippet", ""), keywords) for r in results + ] + num_relevant = sum(relevant_mask) + hit = num_relevant > 0 + + # MRR: reciprocal rank of first relevant chunk + reciprocal_rank = 0.0 + for i, is_relevant in enumerate(relevant_mask): + if is_relevant: + reciprocal_rank = 1.0 / (i + 1) + break + + # Precision@k + precision_at_k = num_relevant / len(results) if results else 0.0 + + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=hit, + reciprocal_rank=reciprocal_rank, + precision_at_k=precision_at_k, + num_retrieved=len(results), + num_relevant=num_relevant, + scores=scores, + ) + + +def print_summary(results: list[QuestionResult], elapsed_s: float) -> None: + total = len(results) + errors = sum(1 for r in results if r.error) + evaluated = total - errors + + if evaluated == 0: + print("No questions evaluated successfully.") + return + + valid = [r for r in results if r.error is None] + hit_rate = sum(r.hit for r in valid) / evaluated + mrr = sum(r.reciprocal_rank for r in valid) / evaluated + avg_precision = sum(r.precision_at_k for r in valid) / evaluated + avg_retrieved = sum(r.num_retrieved for r in valid) / evaluated + avg_relevant = sum(r.num_relevant for r in valid) / evaluated + + print(f"\n{'=' * 60}") + print("Retrieval evaluation results") + print(f"{'=' * 60}") + print(f"Questions: {total} ({errors} errors)") + print(f"Duration: {elapsed_s:.1f}s ({elapsed_s / total:.2f}s/query)") + print("") + print(f"Hit rate: {hit_rate:.1%}") + print(f"MRR: {mrr:.4f}") + print(f"Avg precision: {avg_precision:.4f}") + print(f"Avg retrieved: {avg_retrieved:.1f}") + print(f"Avg relevant: {avg_relevant:.1f}") + + # Breakdown by category + categories = sorted(set(r.category for r in valid)) + if len(categories) > 1: + print("\nBy category:") + for cat in categories: + cat_results = [r for r in valid if r.category == cat] + cat_hit = sum(r.hit for r in cat_results) / len(cat_results) + cat_mrr = sum(r.reciprocal_rank for r in cat_results) / len(cat_results) + print( + f" {cat:<15} hit={cat_hit:.0%} mrr={cat_mrr:.4f} n={len(cat_results)}" + ) + + # Breakdown by language + languages = sorted(set(r.language for r in valid)) + if len(languages) > 1: + print("\nBy language:") + for lang in languages: + lang_results = [r for r in valid if r.language == lang] + lang_hit = sum(r.hit for r in lang_results) / len(lang_results) + lang_mrr = sum(r.reciprocal_rank for r in lang_results) / len(lang_results) + print( + f" {lang:<15} hit={lang_hit:.0%} mrr={lang_mrr:.4f} n={len(lang_results)}" + ) + + # Score distribution + all_scores = [s for r in valid for s in r.scores] + if all_scores: + all_scores.sort() + p25 = all_scores[len(all_scores) // 4] + p50 = all_scores[len(all_scores) // 2] + p75 = all_scores[3 * len(all_scores) // 4] + print( + f"\nScore distribution: min={all_scores[0]:.4f} p25={p25:.4f} p50={p50:.4f} p75={p75:.4f} max={all_scores[-1]:.4f}" + ) + + print(f"{'=' * 60}") + + +def save_results(results: list[QuestionResult], output_path: str) -> None: + with open(output_path, "w") as f: + for r in results: + row = { + "id": r.id, + "question": r.question, + "category": r.category, + "language": r.language, + "hit": r.hit, + "reciprocal_rank": r.reciprocal_rank, + "precision_at_k": r.precision_at_k, + "num_retrieved": r.num_retrieved, + "num_relevant": r.num_relevant, + "scores": r.scores, + } + if r.error: + row["error"] = r.error + f.write(json.dumps(row) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Vektra retrieval evaluation harness") + parser.add_argument( + "--dataset", + default=DEFAULT_DATASET, + help=f"Path to JSONL dataset (default: {DEFAULT_DATASET})", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help=f"Path for JSONL results output (default: {DEFAULT_OUTPUT})", + ) + parser.add_argument( + "--top-k", type=int, default=5, help="Top-k for search (default: 5)" + ) + parser.add_argument( + "--search-mode", + default="hybrid", + choices=["dense", "sparse", "hybrid"], + help="Search mode (default: hybrid)", + ) + parser.add_argument( + "--min-score", + type=float, + default=None, + help="Override VEKTRA_MIN_RELEVANCE_SCORE for this run", + ) + args = parser.parse_args() + + api_url = os.environ.get("VEKTRA_API_URL") + api_key = os.environ.get("VEKTRA_API_KEY") + if not api_key: + # Try .api-key file + api_key_file = Path(".api-key") + if api_key_file.exists(): + api_key = api_key_file.read_text().strip() + + if not api_url or not api_key: + print( + "Error: VEKTRA_API_URL and VEKTRA_API_KEY must be set.", + file=sys.stderr, + ) + sys.exit(1) + + dataset = load_dataset(args.dataset) + if not dataset: + print(f"Error: no entries in {args.dataset}", file=sys.stderr) + sys.exit(1) + + print(f"Loaded {len(dataset)} questions from {args.dataset}") + print(f"API: {api_url} top_k={args.top_k} mode={args.search_mode}") + + client = httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + + results: list[QuestionResult] = [] + t0 = time.monotonic() + for i, entry in enumerate(dataset): + result = evaluate_question(client, entry, args.top_k, args.search_mode) + results.append(result) + status = "HIT" if result.hit else ("ERR" if result.error else "MISS") + print(f" [{i + 1}/{len(dataset)}] {entry['id']} {status}", end="") + if result.scores: + print(f" scores={[round(s, 3) for s in result.scores[:3]]}", end="") + print() + + elapsed = time.monotonic() - t0 + client.close() + + print_summary(results, elapsed) + save_results(results, args.output) + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/tests/eval/dataset.jsonl b/tests/eval/dataset.jsonl new file mode 100644 index 00000000..f903d8aa --- /dev/null +++ b/tests/eval/dataset.jsonl @@ -0,0 +1,5 @@ +{"id": "SEED-IT-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione?", "expected_keywords": ["diritti inviolabili", "Art. 2"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "SEED-IT-02", "question": "Cosa stabilisce l'articolo 3 della Costituzione italiana sull'uguaglianza?", "expected_keywords": ["pari dignit\u00e0 sociale", "senza distinzione"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "SEED-IT-03", "question": "Qual \u00e8 il rapporto tra lo Stato e la Chiesa cattolica secondo la Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "SEED-IT-04", "question": "Come viene regolamentata la libert\u00e0 di stampa?", "expected_keywords": ["stampa", "sequestro", "pubblicazione"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "SEED-IT-05", "question": "What does the Italian Constitution say about the right to work?", "expected_keywords": ["lavoro", "diritto"], "namespace": "default", "category": "factual", "language": "en"} From 54ee4120ce73643dd70ef7f77fd1a44c06b658b4 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Tue, 24 Mar 2026 23:42:06 +0000 Subject: [PATCH 16/33] fix(eval): add --use-query flag to eval_retrieval for reranker evaluation The /api/v1/search endpoint returns raw vector scores without reranker. Add --use-query to call /api/v1/query instead, evaluating retrieval quality after reranking. Handles field name differences between the two endpoints (text_snippet vs snippet). Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/eval_retrieval.py | 62 +++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py index dadc5c58..348bb33e 100644 --- a/scripts/eval_retrieval.py +++ b/scripts/eval_retrieval.py @@ -66,8 +66,14 @@ def evaluate_question( entry: dict, top_k: int, search_mode: str, + use_query: bool = False, ) -> QuestionResult: - """Run a single search query and compute retrieval metrics.""" + """Run a single search/query and compute retrieval metrics. + + When use_query=True, calls /api/v1/query (includes reranker) and + evaluates on the returned sources. Otherwise calls /api/v1/search + (raw vector search, no reranker). + """ question_id = entry["id"] question = entry["question"] keywords = entry.get("expected_keywords", []) @@ -76,15 +82,25 @@ def evaluate_question( language = entry.get("language", "unknown") try: - resp = client.post( - "/api/v1/search", - json={ - "query": question, - "namespace": namespace, - "top_k": top_k, - "search_mode": search_mode, - }, - ) + if use_query: + resp = client.post( + "/api/v1/query", + json={ + "question": question, + "namespace": namespace, + "top_k": top_k, + }, + ) + else: + resp = client.post( + "/api/v1/search", + json={ + "query": question, + "namespace": namespace, + "top_k": top_k, + "search_mode": search_mode, + }, + ) resp.raise_for_status() data = resp.json() except Exception as e: @@ -101,7 +117,8 @@ def evaluate_question( error=str(e), ) - results = data.get("results", []) + # /api/v1/query returns "sources", /api/v1/search returns "results" + results = data.get("sources") or data.get("results") or [] scores = [r["score"] for r in results] if not keywords: @@ -120,8 +137,10 @@ def evaluate_question( ) # Compute which chunks are relevant + # /api/v1/search uses "text_snippet", /api/v1/query uses "snippet" relevant_mask = [ - chunk_matches_keywords(r.get("text_snippet", ""), keywords) for r in results + chunk_matches_keywords(r.get("text_snippet") or r.get("snippet", ""), keywords) + for r in results ] num_relevant = sum(relevant_mask) hit = num_relevant > 0 @@ -258,10 +277,9 @@ def main() -> None: help="Search mode (default: hybrid)", ) parser.add_argument( - "--min-score", - type=float, - default=None, - help="Override VEKTRA_MIN_RELEVANCE_SCORE for this run", + "--use-query", + action="store_true", + help="Use /api/v1/query (with reranker) instead of /api/v1/search", ) args = parser.parse_args() @@ -286,18 +304,24 @@ def main() -> None: sys.exit(1) print(f"Loaded {len(dataset)} questions from {args.dataset}") - print(f"API: {api_url} top_k={args.top_k} mode={args.search_mode}") + mode_info = ( + "query (with reranker)" if args.use_query else f"search mode={args.search_mode}" + ) + print(f"API: {api_url} top_k={args.top_k} {mode_info}") + timeout = 120.0 if args.use_query else 30.0 client = httpx.Client( base_url=api_url, headers={"Authorization": f"Bearer {api_key}"}, - timeout=30.0, + timeout=timeout, ) results: list[QuestionResult] = [] t0 = time.monotonic() for i, entry in enumerate(dataset): - result = evaluate_question(client, entry, args.top_k, args.search_mode) + result = evaluate_question( + client, entry, args.top_k, args.search_mode, use_query=args.use_query + ) results.append(result) status = "HIT" if result.hit else ("ERR" if result.error else "MISS") print(f" [{i + 1}/{len(dataset)}] {entry['id']} {status}", end="") From 08ccf95042ee60c6327cf172f5b83684a4093a69 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 00:14:45 +0000 Subject: [PATCH 17/33] feat(eval): curate 55-question bilingual dataset for retrieval evaluation Dataset covers Italian Constitution and UDHR excerpts: - 20 factual (15 IT + 5 EN) - 15 reasoning (10 IT + 5 EN) - 10 multi-chunk (cross-document) - 10 adversarial (unanswerable from content) Baseline with bge-reranker-v2-m3 at threshold 0.15: factual 90%, reasoning 80%, multi-chunk 10%, adversarial 0% Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/eval/dataset.jsonl | 60 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/eval/dataset.jsonl b/tests/eval/dataset.jsonl index f903d8aa..4ec6b8e1 100644 --- a/tests/eval/dataset.jsonl +++ b/tests/eval/dataset.jsonl @@ -1,5 +1,55 @@ -{"id": "SEED-IT-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione?", "expected_keywords": ["diritti inviolabili", "Art. 2"], "namespace": "default", "category": "factual", "language": "it"} -{"id": "SEED-IT-02", "question": "Cosa stabilisce l'articolo 3 della Costituzione italiana sull'uguaglianza?", "expected_keywords": ["pari dignit\u00e0 sociale", "senza distinzione"], "namespace": "default", "category": "factual", "language": "it"} -{"id": "SEED-IT-03", "question": "Qual \u00e8 il rapporto tra lo Stato e la Chiesa cattolica secondo la Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "default", "category": "reasoning", "language": "it"} -{"id": "SEED-IT-04", "question": "Come viene regolamentata la libert\u00e0 di stampa?", "expected_keywords": ["stampa", "sequestro", "pubblicazione"], "namespace": "default", "category": "factual", "language": "it"} -{"id": "SEED-IT-05", "question": "What does the Italian Constitution say about the right to work?", "expected_keywords": ["lavoro", "diritto"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "IT-F-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione italiana?", "expected_keywords": ["diritti inviolabili", "formazioni sociali"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-02", "question": "Su cosa si fonda la Repubblica italiana?", "expected_keywords": ["fondata sul lavoro"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-03", "question": "Cosa stabilisce l'articolo 3 della Costituzione sull'uguaglianza?", "expected_keywords": ["pari dignita' sociale", "senza distinzione"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-04", "question": "A chi appartiene la sovranita' nella Repubblica italiana?", "expected_keywords": ["sovranita' appartiene al popolo"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-05", "question": "Cosa prevede la Costituzione italiana riguardo al diritto al lavoro?", "expected_keywords": ["diritto al lavoro", "condizioni che rendano effettivo"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-06", "question": "Come e' composta la bandiera della Repubblica italiana?", "expected_keywords": ["tricolore", "verde, bianco e rosso"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-07", "question": "Cosa tutela l'articolo 9 della Costituzione?", "expected_keywords": ["cultura", "ricerca scientifica", "paesaggio"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-08", "question": "Cosa stabilisce la Costituzione sulla liberta' personale?", "expected_keywords": ["liberta' personale", "inviolabile", "autorita' giudiziaria"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-09", "question": "Cosa prevede la Costituzione sulle minoranze linguistiche?", "expected_keywords": ["minoranze linguistiche", "tutela"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-10", "question": "Quali sono le garanzie per la liberta' di corrispondenza?", "expected_keywords": ["corrispondenza", "inviolabili", "autorita' giudiziaria"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-11", "question": "Cosa dice la Costituzione sul diritto di riunione?", "expected_keywords": ["riunirsi pacificamente", "senz'armi"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-12", "question": "Cosa prevede l'articolo 32 sulla salute?", "expected_keywords": ["salute", "fondamentale diritto", "cure gratuite"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-13", "question": "Cosa dice la Costituzione sull'obbligo scolastico?", "expected_keywords": ["istruzione inferiore", "obbligatoria e gratuita", "otto anni"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-14", "question": "Qual e' il rapporto tra Stato e Chiesa nella Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-15", "question": "Cosa prevede l'articolo 21 sulla liberta' di pensiero?", "expected_keywords": ["manifestare liberamente", "pensiero", "stampa"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-R-01", "question": "Qual e' il legame tra il principio di uguaglianza e il diritto al lavoro nella Costituzione?", "expected_keywords": ["uguaglianza", "lavoro"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-02", "question": "Come si conciliano autonomia locale e unita' nazionale nella Costituzione italiana?", "expected_keywords": ["autonomie locali", "una e indivisibile", "decentramento"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-03", "question": "In che modo la Costituzione bilancia liberta' di stampa e intervento giudiziario?", "expected_keywords": ["stampa", "sequestro", "autorita' giudiziaria"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-04", "question": "Quali limiti pone la Costituzione alla liberta' di circolazione?", "expected_keywords": ["circolare", "soggiornare", "sanita'", "sicurezza"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-05", "question": "Come protegge la Costituzione i cittadini capaci ma privi di mezzi economici nel percorso scolastico?", "expected_keywords": ["capaci e meritevoli", "borse di studio"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-06", "question": "In che modo la Costituzione italiana affronta il rapporto tra diritti individuali e solidarieta' sociale?", "expected_keywords": ["diritti inviolabili", "doveri inderogabili", "solidarieta'"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-07", "question": "Quali sono le condizioni per limitare la liberta' personale secondo la Costituzione?", "expected_keywords": ["atto motivato", "autorita' giudiziaria", "casi e modi previsti dalla legge"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-08", "question": "Come affronta la Costituzione la liberta' religiosa per le confessioni diverse dalla cattolica?", "expected_keywords": ["confessioni religiose", "egualmente libere", "intese"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-09", "question": "Qual e' il rapporto tra trattamento sanitario obbligatorio e rispetto della persona nella Costituzione?", "expected_keywords": ["trattamento sanitario", "disposizione di legge", "rispetto della persona umana"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-10", "question": "Come la Costituzione tutela l'ambiente rispetto all'attivita' economica?", "expected_keywords": ["ambiente", "biodiversita'", "attivita' economica", "fini sociali e ambientali"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "EN-F-01", "question": "What does Article 1 of the UDHR say about human beings?", "expected_keywords": ["born free and equal", "dignity and rights"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-02", "question": "What does the UDHR say about slavery?", "expected_keywords": ["slavery", "servitude", "prohibited"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-03", "question": "What rights does Article 19 of the UDHR protect?", "expected_keywords": ["freedom of opinion", "expression", "information"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-04", "question": "What does the UDHR say about education?", "expected_keywords": ["right to education", "free", "compulsory"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-05", "question": "What does Article 9 of the UDHR state about arrest?", "expected_keywords": ["arbitrary arrest", "detention", "exile"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-R-01", "question": "How does the UDHR connect the right to work with human dignity?", "expected_keywords": ["right to work", "remuneration", "dignity"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-02", "question": "What is the relationship between marriage and equality in the UDHR?", "expected_keywords": ["marry", "equal rights", "free and full consent"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-03", "question": "How does the UDHR protect against discrimination?", "expected_keywords": ["without distinction", "equal protection", "discrimination"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-04", "question": "What protections does the UDHR provide for motherhood and childhood?", "expected_keywords": ["motherhood", "childhood", "special care"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-05", "question": "How does the UDHR balance individual rights with government authority?", "expected_keywords": ["will of the people", "elections", "universal", "suffrage"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "MC-01", "question": "Come si confrontano le garanzie sulla liberta' personale nella Costituzione italiana e nella Dichiarazione Universale?", "expected_keywords": ["liberta' personale", "arbitrary arrest"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-02", "question": "Quali analogie esistono tra l'articolo 3 della Costituzione italiana e l'articolo 2 della UDHR sulla non discriminazione?", "expected_keywords": ["senza distinzione", "without distinction"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-03", "question": "How do both the Italian Constitution and the UDHR address freedom of religion?", "expected_keywords": ["fede religiosa", "freedom of thought"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-04", "question": "Quali somiglianze ci sono nel trattamento del diritto all'istruzione tra Costituzione italiana e UDHR?", "expected_keywords": ["istruzione", "education", "obbligatoria"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-05", "question": "How do both documents address the right to work and fair employment?", "expected_keywords": ["diritto al lavoro", "right to work"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-06", "question": "Quali differenze ci sono nella tutela della famiglia tra la Costituzione italiana e la UDHR?", "expected_keywords": ["famiglia", "matrimonio", "family"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-07", "question": "Come affrontano entrambi i documenti il diritto alla privacy e all'inviolabilita' del domicilio?", "expected_keywords": ["domicilio", "privacy", "correspondence"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-08", "question": "What similarities exist in how both documents treat freedom of expression and press?", "expected_keywords": ["pensiero", "expression", "stampa"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-09", "question": "Come entrambi i documenti proteggono il diritto alla salute e a un tenore di vita adeguato?", "expected_keywords": ["salute", "health", "well-being"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-10", "question": "How do the Italian Constitution and UDHR compare on the presumption of innocence and fair trial?", "expected_keywords": ["presumed innocent", "autorita' giudiziaria"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-08", "question": "Cosa dice la Costituzione sul numero massimo di mandati presidenziali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-09", "question": "What penalties does the UDHR prescribe for nations that violate human rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-10", "question": "Come regolamenta la Costituzione italiana l'uso dei social media?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} From d95c31bf847b7c9f519b183b538b1a6e83a2e7d9 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 00:38:34 +0000 Subject: [PATCH 18/33] chore(backlog): add BUG-017, mark BUG-015/016/TECH-002/DEBT-010 completed BUG-017: context window fallback silently truncates prompts for vLLM models not in litellm registry. Discovered via conversation analysis - budget allocator fits only 2/5 chunks when using 4096 default instead of actual 32768. Fix: VEKTRA_LLM_CONTEXT_WINDOW env var + warn on fallback. Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 4fba8e42..bfa5ec1c 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -107,9 +107,9 @@ --- -### BUG-015: Reranker scores discarded after reranking — threshold applied to wrong scores +### BUG-015: ~~Reranker scores discarded after reranking — threshold applied to wrong scores~~ -**Status**: planned | **Priority**: critical | **Created**: 2026-03-24 +**Status**: completed | **Priority**: critical | **Created**: 2026-03-24 | **Completed**: 2026-03-25 **Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` **Context**: `RerankerService.rerank()` (reranker.py:54-60) reorders results but returns the original `SearchResult` objects with their cosine similarity scores intact. The flashrank/cross-encoder scores are used only for ordering, then discarded. The `_apply_retrieval_filter` (pipeline.py:100) then applies `VEKTRA_MIN_RELEVANCE_SCORE=0.3` to these original cosine scores, not the reranker scores. The reranker's relevance judgment and the threshold filter are effectively disconnected: a chunk the reranker ranks highly can still be filtered out if its original cosine similarity was below 0.3. @@ -128,9 +128,9 @@ --- -### BUG-016: English-only reranker produces random scores on Italian content +### BUG-016: ~~English-only reranker produces random scores on Italian content~~ -**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Status**: completed | **Priority**: high | **Created**: 2026-03-24 | **Completed**: 2026-03-25 **Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` **Depends on**: BUG-015 (score propagation must work before reranker swap is meaningful) @@ -165,9 +165,9 @@ VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 --- -### TECH-002: RAG evaluation harness +### TECH-002: ~~RAG evaluation harness~~ -**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Status**: completed | **Priority**: high | **Created**: 2026-03-24 | **Completed**: 2026-03-25 **Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` **Context**: The RAG tuning campaign (2026-03-14-18) used manual testing across 600 queries with qualitative metrics. There is no automated, reproducible way to evaluate retrieval quality when components change (embedding model, reranker, threshold, chunk size). This gap allowed BUG-015 and BUG-016 to go undetected: component interactions were never tested systematically. @@ -192,9 +192,9 @@ VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 --- -### DEBT-010: Recalibrate relevance threshold with empirical data +### DEBT-010: ~~Recalibrate relevance threshold with empirical data~~ -**Status**: planned | **Priority**: medium | **Created**: 2026-03-24 +**Status**: completed | **Priority**: medium | **Created**: 2026-03-24 | **Completed**: 2026-03-25 **Depends on**: BUG-015, BUG-016, TECH-002 **Context**: `VEKTRA_MIN_RELEVANCE_SCORE=0.3` was set per ADR-0021 for cosine similarity with `all-MiniLM-L6-v2` (Phase 1 embedding model). It was never varied in the tuning campaign and not recalibrated when: (a) the embedding model changed to `paraphrase-multilingual-MiniLM-L12-v2`, (b) hybrid search with RRF was enabled, (c) the reranker was added. Different scoring stages produce different distributions (cosine 0.2-0.8 cluster, flashrank bimodal near 0/1, RRF reciprocal). A single threshold cannot serve all correctly. @@ -211,6 +211,33 @@ Literature consensus: use top-k as primary control, low absolute threshold (0.15 --- +### BUG-017: Context window fallback silently truncates prompt — most chunks discarded + +**Status**: planned | **Priority**: high | **Created**: 2026-03-25 + +**Context**: `_context_window_impl()` (pipeline.py:131-136) calls `litellm.get_max_tokens(model)` to determine the context window. For models not in litellm's registry (all local vLLM models like `openai//models/qwen35-27b`), it silently falls back to `_DEFAULT_CONTEXT_WINDOW = 4096`. With Qwen 3.5 27B (actual context: 32768), this causes the token budget allocator to use only ~900 tokens for chunks instead of ~18000. Result: 5 relevant chunks retrieved, but only 2 fit in the prompt, and the LLM produces an incomplete answer. + +**Discovered**: while analyzing conversation `5bf50682` in namespace `ita-100`. User asked "Quali tipi di liberta sono garantiti dalla Costituzione italiana? Elencali tutti con il relativo articolo". Pipeline retrieved 20 candidates, reranker selected 5 (scores 0.78-0.40), threshold kept all 5, but `build_prompt` only included 2 (`chunks_in_prompt: 2`). The answer listed 6 freedoms instead of ~12. + +**Root cause**: no logging or warning when `litellm.get_max_tokens()` fails and the fallback kicks in. The `_count_tokens_impl()` fallback (char/4) is similarly silent. + +**Proposed fix**: +1. Add `VEKTRA_LLM_CONTEXT_WINDOW` env var to LLMConfig (optional int, default None) +2. `_context_window_impl()`: if env var set, use it; else try litellm; on fallback, emit `structlog.warning("context_window_fallback", model=model, default=4096)` +3. `_count_tokens_impl()`: on fallback, emit `structlog.warning("token_count_fallback", model=model)` (once per model, not per call) +4. Same pattern for any other fallback/default in the pipeline + +**Traceability**: ARCH-055 (token budget allocation) + +**Acceptance criteria**: +- [ ] `VEKTRA_LLM_CONTEXT_WINDOW` env var added, used when set +- [ ] Warning logged when context window falls back to default +- [ ] Warning logged when token counting falls back to char/4 +- [ ] Fallback warnings emitted once per model (not per query) to avoid log spam +- [ ] Documentation updated (configuration.md, .env.example) + +--- + ### DEBT-009: Debug logging for rewritten queries **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 @@ -1380,3 +1407,4 @@ The backend stores conversation turns in the database (used for multi-turn query | DEBT-004 (budget ordering) | Phase 2 | Pgvector returns score-desc in practice | | DEBT-005 (disconnect cancel) | Phase 2 | uvicorn handles it implicitly | | DEBT-008 (LRU plaintext cache) | Phase 2 | Replace lru_cache with TTLCache | +| BUG-017 (context window fallback) | Before next release | Silently truncates prompts with vLLM models | From 372e56ce86a6be4ff1d81ad106ca7fb6d914d9fc Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 00:42:12 +0000 Subject: [PATCH 19/33] chore(backlog): add DEBT-011 conversation and query trace observability gaps Four gaps found during post-hoc conversation analysis: no API to read turns (encrypted, DB-only), query traces not persisted for streaming, response_id not populated in conversation_turns, no admin trace lookup. Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index bfa5ec1c..291734af 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -238,6 +238,37 @@ Literature consensus: use top-k as primary control, low absolute threshold (0.15 --- +### DEBT-011: Conversation and query trace observability gaps + +**Status**: planned | **Priority**: medium | **Created**: 2026-03-25 + +**Context**: Diagnosing a conversation (`5bf50682`, namespace `ita-100`) revealed multiple observability gaps that make post-hoc analysis of query behavior difficult: + +1. **No API to read conversation turns**: `GET /api/v1/conversations/{id}` returns metadata (turn_count, namespace, timestamps) but no endpoint exposes the turns themselves. The only way to read them is via direct DB query with `pgp_sym_decrypt()`. + +2. **Query trace not persisted for streaming queries**: When `stream=true`, the trace is emitted via SSE but not saved to `query_traces` table. Non-streaming queries also don't persist traces unless the learn service stores them. The only evidence of a streamed query is a single `query_stream_complete` log line with response_id and duration, no step details. + +3. **response_id not stored in conversation_turns**: The `response_id` column exists but is never populated, making it impossible to correlate a conversation turn with its query trace. + +4. **No admin endpoint for query traces**: Traces can only be retrieved via the learn API (if persisted) or by grepping container logs (which don't survive restarts, see INFRA-005). + +**Proposed approach**: +1. Persist query traces for all queries (not just learn), controlled by a config flag (default: on in development, off in production) +2. Populate `response_id` in conversation_turns when saving a turn +3. Add `GET /api/v1/admin/conversations/{id}/turns` endpoint (admin scope) that decrypts and returns turns +4. Add `GET /api/v1/admin/traces/{response_id}` endpoint for trace lookup + +**Traceability**: ARCH-041 (QueryTrace), ADR-0011 (conversation encryption), ADR-0017 (audit/analytics separation) + +**Acceptance criteria**: +- [ ] Query traces persisted to DB for all pipelines (simple + advanced, sync + stream) +- [ ] `response_id` populated in `conversation_turns` on turn save +- [ ] Admin endpoint to read decrypted conversation turns +- [ ] Admin endpoint to retrieve query trace by response_id +- [ ] Trace persistence configurable (always in dev, opt-in in production) + +--- + ### DEBT-009: Debug logging for rewritten queries **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 @@ -1408,3 +1439,4 @@ The backend stores conversation turns in the database (used for multi-turn query | DEBT-005 (disconnect cancel) | Phase 2 | uvicorn handles it implicitly | | DEBT-008 (LRU plaintext cache) | Phase 2 | Replace lru_cache with TTLCache | | BUG-017 (context window fallback) | Before next release | Silently truncates prompts with vLLM models | +| DEBT-011 (conversation observability) | Post-Phase 2 | Cannot diagnose query behavior post-hoc | From 5d95486fd0a3921171429ca82aab2a707b094953 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 00:43:54 +0000 Subject: [PATCH 20/33] docs(claude): add DB investigation and search vs query pipeline guidance Document key pitfalls: encrypted conversation turns, namespace_id column naming, Qdrant as vector store (not pgvector), and the fundamental difference between /search (raw vector) and /query (full pipeline with reranker/threshold/LLM). Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/CLAUDE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 73cff20e..ba5b7c35 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -12,6 +12,28 @@ Before making **any** API call (curl, httpie, scripts), consult `docs/reference/ Do not construct API calls from memory or guesswork. +## Database investigation + +When querying Postgres directly (psql, DB inspection): +- **Always run `\d table_name` first** to check column names and types. Common pitfalls: `namespace_id` (not `namespace`), `bytea` columns that need decryption, columns that don't exist. +- **Conversation turns are encrypted**: `question` and `answer` are `pgp_sym_encrypt()`'d. To read them: `SELECT pgp_sym_decrypt(question, '') FROM conversation_turns WHERE ...` using `VEKTRA_CONVERSATION_KEY` from `.env`. +- **Postgres holds metadata/text, Qdrant holds vectors**: chunk text is in `document_chunks` (Postgres), but vector search runs against Qdrant (port 10633, collection `vektra`). To inspect vectors or search results, query Qdrant REST API directly. The Qdrant payload uses `namespace_id` as the namespace filter field. + +## Query pipeline vs search endpoint + +These are fundamentally different: + +| | `/api/v1/search` | `/api/v1/query` | +|--|--|--| +| What it does | Raw vector search (Qdrant only) | Full RAG pipeline | +| Reranker | No | Yes | +| Threshold filter | No | Yes | +| LLM call | No | Yes | +| Field for question | `query` | `question` | +| Response field | `results[].text_snippet` | `sources[].snippet` | + +When investigating retrieval quality, use `/api/v1/query` to see the full pipeline behavior. Use `/api/v1/search` only to inspect raw vector similarity. + ## Spec2Ship Commands - `/s2s:specs` - Define requirements via roundtable From ffbc9b2d9e7f5dd1ccc08c6076e51674ff06fd51 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 11:11:15 +0000 Subject: [PATCH 21/33] fix(core): add VEKTRA_LLM_CONTEXT_WINDOW and warn on fallback (BUG-017) litellm.get_max_tokens() silently falls back to 4096 for models not in its registry (all local vLLM models), causing the token budget allocator to discard most retrieved chunks. Add VEKTRA_LLM_CONTEXT_WINDOW env var as explicit override, and emit structlog warnings (once per model) when context window or token count fall back to defaults. Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 1 + docs/reference/configuration.md | 1 + .../src/vektra_core/advanced_pipeline.py | 4 +- vektra-core/src/vektra_core/pipeline.py | 42 +++++++++++-- vektra-core/tests/test_pipeline.py | 60 +++++++++++++++++++ vektra-shared/src/vektra_shared/config.py | 5 ++ 6 files changed, 107 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index d5541929..91d888e2 100644 --- a/.env.example +++ b/.env.example @@ -112,6 +112,7 @@ # Fallback model when primary times out. # VEKTRA_LLM_FALLBACK_MODEL= # VEKTRA_LLM_FALLBACK_TIMEOUT_MS=60000 +# VEKTRA_LLM_CONTEXT_WINDOW= # Required for models not in litellm registry (e.g. local vLLM). Example: 32768 # VEKTRA_LLM_CONTEXT_ONLY_ENABLED=true # -------------------------------------------------------------------------- diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index ac6802a5..68af7296 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -29,6 +29,7 @@ If using a cloud provider, also set the corresponding API key (see [LLM provider | `VEKTRA_LLM_API_BASE` | str | - | Custom API base URL for OpenAI-compatible providers (e.g., vLLM, LMStudio) | | `VEKTRA_LLM_FALLBACK_MODEL` | str | - | Fallback model when the primary times out | | `VEKTRA_LLM_FALLBACK_TIMEOUT_MS` | int | `60000` | Milliseconds before switching to fallback model | +| `VEKTRA_LLM_CONTEXT_WINDOW` | int | - | Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup with 4096 fallback (warns on fallback) | | `VEKTRA_LLM_CONTEXT_ONLY_ENABLED` | bool | `true` | Return raw chunks without LLM synthesis when both models fail | | `VEKTRA_STARTUP_LLM_CHECK` | bool | `true` | Test LLM connectivity at startup (warning-only, not fatal) | diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 2b753db0..cec6a638 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -103,7 +103,9 @@ def _count_tokens(self, text: str) -> int: return _count_tokens_impl(self._llm, self._llm_config.provider, text) def _context_window(self) -> int: - return _context_window_impl(self._llm_config.provider) + return _context_window_impl( + self._llm_config.provider, self._llm_config.context_window + ) async def _call_llm_with_fallback( self, diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index 4f1cdc92..540573fb 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -120,20 +120,50 @@ def _apply_retrieval_filter( # --------------------------------------------------------------------------- +_token_count_fallback_warned: set[str] = set() +_context_window_fallback_warned: set[str] = set() + + def _count_tokens_impl(llm: LLMProvider, model: str, text: str) -> int: """Count tokens using the LLM provider, with char/4 fallback.""" try: return llm.count_tokens(text, model) except Exception: + if model not in _token_count_fallback_warned: + _token_count_fallback_warned.add(model) + log.warning( + "token_count_fallback", + model=model, + method="char_div_4", + hint="Set VEKTRA_LLM_CONTEXT_WINDOW to ensure correct token budget allocation", + ) return max(1, len(text) // 4) -def _context_window_impl(model: str) -> int: - """Get context window size from litellm, with default fallback.""" +def _context_window_impl(model: str, configured_window: int | None = None) -> int: + """Get context window size, with fallback chain and warnings. + + Priority: configured_window (env var) > litellm lookup > default 4096. + """ + if configured_window is not None: + return configured_window + try: - return litellm.get_max_tokens(model) or _DEFAULT_CONTEXT_WINDOW + result = litellm.get_max_tokens(model) + if result: + return result except Exception: - return _DEFAULT_CONTEXT_WINDOW + pass + + if model not in _context_window_fallback_warned: + _context_window_fallback_warned.add(model) + log.warning( + "context_window_fallback", + model=model, + default=_DEFAULT_CONTEXT_WINDOW, + hint="Model not in litellm registry. Set VEKTRA_LLM_CONTEXT_WINDOW to the correct value", + ) + return _DEFAULT_CONTEXT_WINDOW async def _call_llm_with_fallback_impl( @@ -237,7 +267,9 @@ def _count_tokens(self, text: str) -> int: return _count_tokens_impl(self._llm, self._llm_config.provider, text) def _context_window(self) -> int: - return _context_window_impl(self._llm_config.provider) + return _context_window_impl( + self._llm_config.provider, self._llm_config.context_window + ) async def _call_llm_with_fallback( self, diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index 907aa6bc..935f4b47 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -7,6 +7,10 @@ from vektra_core.pipeline import ( SimpleQueryPipeline, _apply_retrieval_filter, + _context_window_fallback_warned, + _context_window_impl, + _count_tokens_impl, + _token_count_fallback_warned, _token_overlap_ratio, ) from vektra_core.templates import TemplateRenderer @@ -466,3 +470,59 @@ async def _failing_stream(): assert "error" in types assert "trace" in types assert types[-1] == "done" + + +# --------------------------------------------------------------------------- +# _context_window_impl (BUG-017) +# --------------------------------------------------------------------------- + + +def test_context_window_uses_configured_value(): + """Configured context_window takes priority over litellm lookup.""" + result = _context_window_impl("nonexistent/model", configured_window=32768) + assert result == 32768 + + +def test_context_window_fallback_to_default(): + """Unknown model without configured window falls back to 4096 with warning.""" + _context_window_fallback_warned.discard("test/unknown-model-ctx") + result = _context_window_impl("test/unknown-model-ctx") + assert result == 4096 + assert "test/unknown-model-ctx" in _context_window_fallback_warned + + +def test_context_window_fallback_warns_once(capsys): + """Fallback warning is emitted only once per model.""" + _context_window_fallback_warned.discard("test/warn-once-model") + _context_window_impl("test/warn-once-model") + capsys.readouterr() # clear first warning + _context_window_impl("test/warn-once-model") + captured = capsys.readouterr() + assert "context_window_fallback" not in captured.out + + +# --------------------------------------------------------------------------- +# _count_tokens_impl fallback warning (BUG-017) +# --------------------------------------------------------------------------- + + +def test_count_tokens_fallback_warns(): + """Token count fallback emits warning on first use per model.""" + _token_count_fallback_warned.discard("test/token-fallback-model") + mock_llm = MagicMock() + mock_llm.count_tokens.side_effect = Exception("unsupported") + result = _count_tokens_impl(mock_llm, "test/token-fallback-model", "hello world") + assert result == max(1, len("hello world") // 4) + assert "test/token-fallback-model" in _token_count_fallback_warned + + +def test_count_tokens_fallback_warns_once(capsys): + """Token count fallback warning only once per model.""" + _token_count_fallback_warned.discard("test/token-once-model") + mock_llm = MagicMock() + mock_llm.count_tokens.side_effect = Exception("unsupported") + _count_tokens_impl(mock_llm, "test/token-once-model", "first") + capsys.readouterr() # clear first warning + _count_tokens_impl(mock_llm, "test/token-once-model", "second") + captured = capsys.readouterr() + assert "token_count_fallback" not in captured.out diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 214a49d3..dcef80e8 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -48,6 +48,11 @@ class LLMConfig(BaseSettings): alias="VEKTRA_LLM_FALLBACK_TIMEOUT_MS", description="Timeout in ms before switching to fallback model.", ) + context_window: int | None = Field( + None, + alias="VEKTRA_LLM_CONTEXT_WINDOW", + description="Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup is attempted with a 4096-token fallback.", + ) context_only_enabled: bool = Field( True, alias="VEKTRA_LLM_CONTEXT_ONLY_ENABLED", From 6685f7ff516dc17ef2b6379cdbb08ac69e3be76f Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 11:11:25 +0000 Subject: [PATCH 22/33] chore(backlog): mark BUG-017 completed Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 291734af..7947ba9f 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -211,9 +211,9 @@ Literature consensus: use top-k as primary control, low absolute threshold (0.15 --- -### BUG-017: Context window fallback silently truncates prompt — most chunks discarded +### BUG-017: ~~Context window fallback silently truncates prompt — most chunks discarded~~ -**Status**: planned | **Priority**: high | **Created**: 2026-03-25 +**Status**: completed | **Priority**: high | **Created**: 2026-03-25 | **Completed**: 2026-03-25 **Context**: `_context_window_impl()` (pipeline.py:131-136) calls `litellm.get_max_tokens(model)` to determine the context window. For models not in litellm's registry (all local vLLM models like `openai//models/qwen35-27b`), it silently falls back to `_DEFAULT_CONTEXT_WINDOW = 4096`. With Qwen 3.5 27B (actual context: 32768), this causes the token budget allocator to use only ~900 tokens for chunks instead of ~18000. Result: 5 relevant chunks retrieved, but only 2 fit in the prompt, and the LLM produces an incomplete answer. From f31b5886163c107afcf8a61670b6f501f56dcc6d Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 18:54:20 +0000 Subject: [PATCH 23/33] fix(config): forward VEKTRA_LLM_CONTEXT_WINDOW through VektraSettings context_window was defined in LLMConfig but not stored in VektraSettings or forwarded by as_llm_config(). Worked by accident via pydantic-settings env fallback, but broke when passing values programmatically. Add llm_context_window field and forward it explicitly. Also add ge=1 validation to reject invalid values. Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-shared/src/vektra_shared/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index dcef80e8..a8d88cb4 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -50,6 +50,7 @@ class LLMConfig(BaseSettings): ) context_window: int | None = Field( None, + ge=1, alias="VEKTRA_LLM_CONTEXT_WINDOW", description="Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup is attempted with a 4096-token fallback.", ) @@ -428,6 +429,7 @@ class VektraSettings(BaseSettings): llm_api_key: str | None = Field(None, alias="VEKTRA_LLM_API_KEY") llm_api_base: str | None = Field(None, alias="VEKTRA_LLM_API_BASE") llm_extra_body: dict[str, Any] | None = Field(None, alias="VEKTRA_LLM_EXTRA_BODY") + llm_context_window: int | None = Field(None, alias="VEKTRA_LLM_CONTEXT_WINDOW") llm_fallback_model: str | None = Field(None, alias="VEKTRA_LLM_FALLBACK_MODEL") llm_fallback_timeout_ms: int = Field(60000, alias="VEKTRA_LLM_FALLBACK_TIMEOUT_MS") llm_context_only_enabled: bool = Field( @@ -537,6 +539,7 @@ def as_llm_config(self) -> LLMConfig: "VEKTRA_LLM_API_KEY": self.llm_api_key, "VEKTRA_LLM_API_BASE": self.llm_api_base, "VEKTRA_LLM_EXTRA_BODY": self.llm_extra_body, + "VEKTRA_LLM_CONTEXT_WINDOW": self.llm_context_window, "VEKTRA_LLM_FALLBACK_MODEL": self.llm_fallback_model, "VEKTRA_LLM_FALLBACK_TIMEOUT_MS": self.llm_fallback_timeout_ms, "VEKTRA_LLM_CONTEXT_ONLY_ENABLED": self.llm_context_only_enabled, From 80189877a993ae4d8e79c00a5d639c5772b9b1b8 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 18:56:24 +0000 Subject: [PATCH 24/33] fix(eval): exclude keyword-less entries from retrieval metrics Adversarial questions (no expected_keywords) were averaged into hit rate, MRR, and precision, biasing all metrics downward. Add has_ground_truth flag and compute metrics only over scored entries. Unscored count shown separately in summary. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/eval_retrieval.py | 87 +++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py index 348bb33e..ca7f53e4 100644 --- a/scripts/eval_retrieval.py +++ b/scripts/eval_retrieval.py @@ -37,6 +37,7 @@ class QuestionResult: precision_at_k: float # relevant chunks / total chunks num_retrieved: int num_relevant: int + has_ground_truth: bool = True # False for entries without expected_keywords scores: list[float] = field(default_factory=list) error: str | None = None @@ -133,6 +134,7 @@ def evaluate_question( precision_at_k=0.0, num_retrieved=len(results), num_relevant=0, + has_ground_truth=False, scores=scores, ) @@ -179,47 +181,61 @@ def print_summary(results: list[QuestionResult], elapsed_s: float) -> None: return valid = [r for r in results if r.error is None] - hit_rate = sum(r.hit for r in valid) / evaluated - mrr = sum(r.reciprocal_rank for r in valid) / evaluated - avg_precision = sum(r.precision_at_k for r in valid) / evaluated - avg_retrieved = sum(r.num_retrieved for r in valid) / evaluated - avg_relevant = sum(r.num_relevant for r in valid) / evaluated + # Separate scored (have ground truth keywords) from unscored (adversarial) + scored = [r for r in valid if r.has_ground_truth] + unscored = [r for r in valid if not r.has_ground_truth] print(f"\n{'=' * 60}") print("Retrieval evaluation results") print(f"{'=' * 60}") - print(f"Questions: {total} ({errors} errors)") + print( + f"Questions: {total} ({errors} errors, {len(unscored)} without ground truth)" + ) print(f"Duration: {elapsed_s:.1f}s ({elapsed_s / total:.2f}s/query)") - print("") - print(f"Hit rate: {hit_rate:.1%}") - print(f"MRR: {mrr:.4f}") - print(f"Avg precision: {avg_precision:.4f}") - print(f"Avg retrieved: {avg_retrieved:.1f}") - print(f"Avg relevant: {avg_relevant:.1f}") - - # Breakdown by category - categories = sorted(set(r.category for r in valid)) - if len(categories) > 1: - print("\nBy category:") - for cat in categories: - cat_results = [r for r in valid if r.category == cat] - cat_hit = sum(r.hit for r in cat_results) / len(cat_results) - cat_mrr = sum(r.reciprocal_rank for r in cat_results) / len(cat_results) - print( - f" {cat:<15} hit={cat_hit:.0%} mrr={cat_mrr:.4f} n={len(cat_results)}" - ) - # Breakdown by language - languages = sorted(set(r.language for r in valid)) - if len(languages) > 1: - print("\nBy language:") - for lang in languages: - lang_results = [r for r in valid if r.language == lang] - lang_hit = sum(r.hit for r in lang_results) / len(lang_results) - lang_mrr = sum(r.reciprocal_rank for r in lang_results) / len(lang_results) - print( - f" {lang:<15} hit={lang_hit:.0%} mrr={lang_mrr:.4f} n={len(lang_results)}" - ) + if scored: + hit_rate = sum(r.hit for r in scored) / len(scored) + mrr = sum(r.reciprocal_rank for r in scored) / len(scored) + avg_precision = sum(r.precision_at_k for r in scored) / len(scored) + avg_retrieved = sum(r.num_retrieved for r in scored) / len(scored) + avg_relevant = sum(r.num_relevant for r in scored) / len(scored) + + print(f"\nScored ({len(scored)} questions with ground truth):") + print(f" Hit rate: {hit_rate:.1%}") + print(f" MRR: {mrr:.4f}") + print(f" Avg precision: {avg_precision:.4f}") + print(f" Avg retrieved: {avg_retrieved:.1f}") + print(f" Avg relevant: {avg_relevant:.1f}") + else: + print("\nNo scored questions (all entries lack ground truth keywords).") + + # Breakdown by category (scored only) + if scored: + categories = sorted(set(r.category for r in scored)) + if len(categories) > 1: + print("\n By category:") + for cat in categories: + cat_results = [r for r in scored if r.category == cat] + cat_hit = sum(r.hit for r in cat_results) / len(cat_results) + cat_mrr = sum(r.reciprocal_rank for r in cat_results) / len(cat_results) + print( + f" {cat:<15} hit={cat_hit:.0%} mrr={cat_mrr:.4f} n={len(cat_results)}" + ) + + # Breakdown by language (scored only) + if scored: + languages = sorted(set(r.language for r in scored)) + if len(languages) > 1: + print("\n By language:") + for lang in languages: + lang_results = [r for r in scored if r.language == lang] + lang_hit = sum(r.hit for r in lang_results) / len(lang_results) + lang_mrr = sum(r.reciprocal_rank for r in lang_results) / len( + lang_results + ) + print( + f" {lang:<15} hit={lang_hit:.0%} mrr={lang_mrr:.4f} n={len(lang_results)}" + ) # Score distribution all_scores = [s for r in valid for s in r.scores] @@ -248,6 +264,7 @@ def save_results(results: list[QuestionResult], output_path: str) -> None: "precision_at_k": r.precision_at_k, "num_retrieved": r.num_retrieved, "num_relevant": r.num_relevant, + "has_ground_truth": r.has_ground_truth, "scores": r.scores, } if r.error: From ba56947eadcc50e73650a7181cf221532cd44b65 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 18:57:01 +0000 Subject: [PATCH 25/33] test(core): assert first warning emitted in warn-once tests Tests only verified the second call was silent but not that the first call actually emitted the warning. If logging broke entirely, both tests would still pass. Co-Authored-By: Claude Opus 4.6 (1M context) --- vektra-core/tests/test_pipeline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index 935f4b47..6c6b1900 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -492,13 +492,14 @@ def test_context_window_fallback_to_default(): def test_context_window_fallback_warns_once(capsys): - """Fallback warning is emitted only once per model.""" + """Fallback warning is emitted on first call, silent on second.""" _context_window_fallback_warned.discard("test/warn-once-model") _context_window_impl("test/warn-once-model") - capsys.readouterr() # clear first warning + first = capsys.readouterr() + assert "context_window_fallback" in first.out _context_window_impl("test/warn-once-model") - captured = capsys.readouterr() - assert "context_window_fallback" not in captured.out + second = capsys.readouterr() + assert "context_window_fallback" not in second.out # --------------------------------------------------------------------------- @@ -517,12 +518,13 @@ def test_count_tokens_fallback_warns(): def test_count_tokens_fallback_warns_once(capsys): - """Token count fallback warning only once per model.""" + """Token count fallback warning emitted on first call, silent on second.""" _token_count_fallback_warned.discard("test/token-once-model") mock_llm = MagicMock() mock_llm.count_tokens.side_effect = Exception("unsupported") _count_tokens_impl(mock_llm, "test/token-once-model", "first") - capsys.readouterr() # clear first warning + first = capsys.readouterr() + assert "token_count_fallback" in first.out _count_tokens_impl(mock_llm, "test/token-once-model", "second") - captured = capsys.readouterr() - assert "token_count_fallback" not in captured.out + second = capsys.readouterr() + assert "token_count_fallback" not in second.out From 777b4dbab5d14206645741bdc4d08787d0103e2d Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 18:57:26 +0000 Subject: [PATCH 26/33] fix(eval): do not mark transport errors as no_relevant_context API/network errors were incorrectly classified as no_relevant_context, inflating the no-context count and omitting duration_ms. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/eval_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/eval_e2e.py b/scripts/eval_e2e.py index a818fa36..9f4ad808 100644 --- a/scripts/eval_e2e.py +++ b/scripts/eval_e2e.py @@ -83,8 +83,9 @@ def evaluate_question(client: httpx.Client, entry: dict, top_k: int) -> E2EResul category=category, language=language, answer=None, - no_relevant_context=True, + no_relevant_context=False, num_sources=0, + duration_ms=(time.monotonic() - t0) * 1000, error=str(e), ) From fe910a4f904188f2a856d8c14f32573c31bec50c Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 18:57:51 +0000 Subject: [PATCH 27/33] docs(config): add VEKTRA_LLM_EXTRA_BODY to LLM reference table Missing from the primary config matrix, only documented in the local vLLM section. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/reference/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 68af7296..093a3f30 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -27,6 +27,7 @@ If using a cloud provider, also set the corresponding API key (see [LLM provider | `VEKTRA_LLM_PROVIDER` | str | (required) | LLM model in litellm format | | `VEKTRA_LLM_API_KEY` | str | - | API key for the LLM provider. Not needed for Ollama. | | `VEKTRA_LLM_API_BASE` | str | - | Custom API base URL for OpenAI-compatible providers (e.g., vLLM, LMStudio) | +| `VEKTRA_LLM_EXTRA_BODY` | json | - | Extra JSON body passed to litellm (e.g. `{"chat_template_kwargs": {"enable_thinking": false}}` for vLLM thinking models) | | `VEKTRA_LLM_FALLBACK_MODEL` | str | - | Fallback model when the primary times out | | `VEKTRA_LLM_FALLBACK_TIMEOUT_MS` | int | `60000` | Milliseconds before switching to fallback model | | `VEKTRA_LLM_CONTEXT_WINDOW` | int | - | Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup with 4096 fallback (warns on fallback) | From bca0e6aa85588275e54b3df74b03435faf9ba237 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 19:02:13 +0000 Subject: [PATCH 28/33] docs(claude): remove hardcoded Qdrant port, clarify search vs query Reference config env vars instead of hardcoding port 10633 and collection name. Clarify wording for when to use each endpoint. Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index ba5b7c35..984ba8ac 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -17,7 +17,7 @@ Do not construct API calls from memory or guesswork. When querying Postgres directly (psql, DB inspection): - **Always run `\d table_name` first** to check column names and types. Common pitfalls: `namespace_id` (not `namespace`), `bytea` columns that need decryption, columns that don't exist. - **Conversation turns are encrypted**: `question` and `answer` are `pgp_sym_encrypt()`'d. To read them: `SELECT pgp_sym_decrypt(question, '') FROM conversation_turns WHERE ...` using `VEKTRA_CONVERSATION_KEY` from `.env`. -- **Postgres holds metadata/text, Qdrant holds vectors**: chunk text is in `document_chunks` (Postgres), but vector search runs against Qdrant (port 10633, collection `vektra`). To inspect vectors or search results, query Qdrant REST API directly. The Qdrant payload uses `namespace_id` as the namespace filter field. +- **Postgres holds metadata/text, Qdrant holds vectors**: chunk text is in `document_chunks` (Postgres), but vector search runs against Qdrant (check `VEKTRA_QDRANT_URL` and `VEKTRA_QDRANT_COLLECTION` in config for host/port/collection). To inspect vectors or search results, query Qdrant REST API directly. The Qdrant payload uses `namespace_id` as the namespace filter field. ## Query pipeline vs search endpoint @@ -32,7 +32,7 @@ These are fundamentally different: | Field for question | `query` | `question` | | Response field | `results[].text_snippet` | `sources[].snippet` | -When investigating retrieval quality, use `/api/v1/query` to see the full pipeline behavior. Use `/api/v1/search` only to inspect raw vector similarity. +When investigating **end-to-end answer quality** (retrieval + reranking + LLM), use `/api/v1/query`. When investigating **raw retrieval quality** (vector similarity only, no reranker), use `/api/v1/search`. ## Spec2Ship Commands From 1a8eb1488f9d6fb8a144a6b09afae52c2dbfa760 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 19:02:38 +0000 Subject: [PATCH 29/33] fix(build): avoid redundant uv sync when INSTALL_UNSTRUCTURED=true Was running uv sync twice: once without --extra ocr, then again with it. Restructured to run once with the correct extras. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 251f7bea..a0eb597c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,9 +70,10 @@ COPY vektra-app/src vektra-app/src # Include optional extras for Phase 2 vector store and sparse search support. # When INSTALL_UNSTRUCTURED=true, also install the Unstructured PDF extractor. ARG INSTALL_UNSTRUCTURED=false -RUN uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant \ - && if [ "$INSTALL_UNSTRUCTURED" = "true" ]; then \ +RUN if [ "$INSTALL_UNSTRUCTURED" = "true" ]; then \ uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant --extra ocr; \ + else \ + uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant; \ fi # -------------------------------------------------------------------------- From 026b870a8f78508005b140be67b4a58a17f56428 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 19:23:27 +0000 Subject: [PATCH 30/33] chore(backlog): add BUG-018 SSE conversation_id not returned to client Streaming path does not emit server-generated conversation_id in any SSE event. Direct API consumers using SSE without pre-generating an ID cannot discover which conversation to continue. Co-Authored-By: Claude Opus 4.6 (1M context) --- .s2s/BACKLOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 7947ba9f..06201c9c 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -238,9 +238,31 @@ Literature consensus: use top-k as primary control, low absolute threshold (0.15 --- +### BUG-018: SSE streaming path does not return server-generated conversation_id + +**Status**: planned | **Priority**: medium | **Created**: 2026-03-25 + +**Context**: When a client calls `POST /api/v1/query` with `stream=true` and no `conversation_id`, the server creates a conversation row and passes the ID to the pipeline. However, the SSE event stream never emits this ID back to the client. The non-streaming path returns it in the JSON response (`conversation_id` field), but the streaming path has no equivalent. + +A client using SSE without generating its own `conversation_id` cannot discover which ID to use for subsequent turns, breaking multi-turn conversations. + +**Current impact**: low. The widget always generates `conversation_id` client-side, so production is unaffected. The bug affects direct API consumers using SSE without pre-generating an ID. + +**Proposed fix**: emit the `conversation_id` in the first SSE event (e.g. a `metadata` event before tokens start) or in the `done` event payload. + +**Traceability**: BUG-014 (conversation persistence), DEBT-011 (observability gaps) + +**Acceptance criteria**: +- [ ] SSE stream includes `conversation_id` in an event accessible before or after token streaming +- [ ] Client can extract the ID and use it for follow-up queries +- [ ] Non-streaming path behavior unchanged + +--- + ### DEBT-011: Conversation and query trace observability gaps **Status**: planned | **Priority**: medium | **Created**: 2026-03-25 +**Related**: BUG-018 (SSE conversation_id) **Context**: Diagnosing a conversation (`5bf50682`, namespace `ita-100`) revealed multiple observability gaps that make post-hoc analysis of query behavior difficult: @@ -1439,4 +1461,5 @@ The backend stores conversation turns in the database (used for multi-turn query | DEBT-005 (disconnect cancel) | Phase 2 | uvicorn handles it implicitly | | DEBT-008 (LRU plaintext cache) | Phase 2 | Replace lru_cache with TTLCache | | BUG-017 (context window fallback) | Before next release | Silently truncates prompts with vLLM models | +| BUG-018 (SSE conversation_id) | Before next release | Streaming clients can't discover server-generated ID | | DEBT-011 (conversation observability) | Post-Phase 2 | Cannot diagnose query behavior post-hoc | From 35cfb57b7205179fb231e5e2ab6a4f1295e4683c Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 20:56:49 +0000 Subject: [PATCH 31/33] fix(ci): reduce perf measurement queries from 100 to 20 Cross-encoder reranker (bge-m3) takes ~4s per query vs ~50ms for flashrank. 100 queries caused the integration job to exceed its 20-minute timeout. 20 queries is sufficient for p50/p95 on a non-blocking warn-only measurement. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/nfr/test_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nfr/test_performance.py b/tests/nfr/test_performance.py index a3ffbf01..a088da5a 100644 --- a/tests/nfr/test_performance.py +++ b/tests/nfr/test_performance.py @@ -22,7 +22,7 @@ _NOTE_THRESHOLD = 1.2 # 20% above target -> note _WARN_THRESHOLD = 1.5 # 50% above target -> warn -_NUM_QUERIES = 100 +_NUM_QUERIES = 20 def test_query_latency_measurement(api: httpx.Client, admin_key: str) -> None: From a483f1ed5b2c94652bf02c98d893c5faf8197a78 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 21:03:33 +0000 Subject: [PATCH 32/33] fix(ci): skip latency measurement on PR, run only post-merge Query latency measurement (100 queries with cross-encoder reranker) takes ~7 minutes and is warn-only. No reason to run on every PR push. Now runs only on push to develop/main (post-merge). Restored 100 queries and raised job timeout to 30 minutes to accommodate. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration.yml | 6 +++--- tests/nfr/test_performance.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 49fcc3dc..11613cb3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ jobs: integration: name: Integration tests + NFR gates runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 @@ -72,8 +72,8 @@ jobs: VEKTRA_BOOTSTRAP_KEY: ci-bootstrap-key STARTUP_MS: ${{ steps.startup.outputs.startup_ms }} - - name: Query latency measurement (warn only) - if: always() && !cancelled() + - name: Query latency measurement (warn only, post-merge only) + if: always() && !cancelled() && github.event_name == 'push' continue-on-error: true run: | uv run pytest tests/nfr/test_performance.py \ diff --git a/tests/nfr/test_performance.py b/tests/nfr/test_performance.py index a088da5a..a3ffbf01 100644 --- a/tests/nfr/test_performance.py +++ b/tests/nfr/test_performance.py @@ -22,7 +22,7 @@ _NOTE_THRESHOLD = 1.2 # 20% above target -> note _WARN_THRESHOLD = 1.5 # 50% above target -> warn -_NUM_QUERIES = 20 +_NUM_QUERIES = 100 def test_query_latency_measurement(api: httpx.Client, admin_key: str) -> None: From 4f2fa6a913ab6095437029cbd99559a93ffff554 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Wed, 25 Mar 2026 21:17:03 +0000 Subject: [PATCH 33/33] style(eval): minor fixes from review round 2 - Add shell quoting note for VEKTRA_LLM_EXTRA_BODY JSON values - Set has_ground_truth=False on error results (defensive) - Use ensure_ascii=False in eval_retrieval JSONL output (matches eval_e2e) - Add VEKTRA_LLM_CONTEXT_WINDOW to vLLM example block Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/reference/configuration.md | 3 +++ scripts/eval_retrieval.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 093a3f30..e16cdbd4 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -46,10 +46,13 @@ Provider-specific environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) VEKTRA_LLM_PROVIDER=openai//models/your-model-name VEKTRA_LLM_API_KEY= VEKTRA_LLM_API_BASE=http://:8000/v1 +VEKTRA_LLM_CONTEXT_WINDOW=32768 # For models with thinking mode (e.g. Qwen3.5, DeepSeek-R1), disable it: VEKTRA_LLM_EXTRA_BODY={"chat_template_kwargs": {"enable_thinking": false}} ``` +In `.env` files (Docker Compose), JSON values do not need quoting. In shell, use single quotes: `VEKTRA_LLM_EXTRA_BODY='{"chat_template_kwargs": {"enable_thinking": false}}'`. + The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35-27b`). ## Embedding diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py index ca7f53e4..e5e35d98 100644 --- a/scripts/eval_retrieval.py +++ b/scripts/eval_retrieval.py @@ -115,6 +115,7 @@ def evaluate_question( precision_at_k=0.0, num_retrieved=0, num_relevant=0, + has_ground_truth=False, error=str(e), ) @@ -269,7 +270,7 @@ def save_results(results: list[QuestionResult], output_path: str) -> None: } if r.error: row["error"] = r.error - f.write(json.dumps(row) + "\n") + f.write(json.dumps(row, ensure_ascii=False) + "\n") def main() -> None: