diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 83aecd57..984ba8ac 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -2,6 +2,38 @@ @../.s2s/CONTEXT.md +## API interaction + +Before making **any** API call (curl, httpie, scripts), consult `docs/reference/api.md` or the live OpenAPI spec at `/openapi.json` to verify: +- Authentication method and header format +- Parameter names, types, and whether they are query, path, or body params +- Request body field names (e.g. `question` vs `query`) +- Correct endpoint for the task (e.g. `/api/v1/query` for RAG, `/api/v1/search` for raw vector search) + +Do not construct API calls from memory or guesswork. + +## Database investigation + +When querying Postgres directly (psql, DB inspection): +- **Always run `\d table_name` first** to check column names and types. Common pitfalls: `namespace_id` (not `namespace`), `bytea` columns that need decryption, columns that don't exist. +- **Conversation turns are encrypted**: `question` and `answer` are `pgp_sym_encrypt()`'d. To read them: `SELECT pgp_sym_decrypt(question, '') FROM conversation_turns WHERE ...` using `VEKTRA_CONVERSATION_KEY` from `.env`. +- **Postgres holds metadata/text, Qdrant holds vectors**: chunk text is in `document_chunks` (Postgres), but vector search runs against Qdrant (check `VEKTRA_QDRANT_URL` and `VEKTRA_QDRANT_COLLECTION` in config for host/port/collection). To inspect vectors or search results, query Qdrant REST API directly. The Qdrant payload uses `namespace_id` as the namespace filter field. + +## Query pipeline vs search endpoint + +These are fundamentally different: + +| | `/api/v1/search` | `/api/v1/query` | +|--|--|--| +| What it does | Raw vector search (Qdrant only) | Full RAG pipeline | +| Reranker | No | Yes | +| Threshold filter | No | Yes | +| LLM call | No | Yes | +| Field for question | `query` | `question` | +| Response field | `results[].text_snippet` | `sources[].snippet` | + +When investigating **end-to-end answer quality** (retrieval + reranking + LLM), use `/api/v1/query`. When investigating **raw retrieval quality** (vector similarity only, no reranker), use `/api/v1/search`. + ## Spec2Ship Commands - `/s2s:specs` - Define requirements via roundtable diff --git a/.env.example b/.env.example index 67e08912..91d888e2 100644 --- a/.env.example +++ b/.env.example @@ -77,7 +77,7 @@ # -------------------------------------------------------------------------- # VEKTRA_QUERY_PIPELINE=advanced -# VEKTRA_MIN_RELEVANCE_SCORE=0.3 +# VEKTRA_MIN_RELEVANCE_SCORE=0.15 # VEKTRA_CHUNK_DEDUP_ENABLED=true # VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 @@ -89,8 +89,8 @@ # Reranking # VEKTRA_RERANK_ENABLED=true -# VEKTRA_RERANK_PROVIDER=flashrank -# VEKTRA_RERANK_MODEL= +# VEKTRA_RERANK_PROVIDER=cross-encoder +# VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 # VEKTRA_RERANK_TOP_K=5 # -------------------------------------------------------------------------- @@ -112,6 +112,7 @@ # Fallback model when primary times out. # VEKTRA_LLM_FALLBACK_MODEL= # VEKTRA_LLM_FALLBACK_TIMEOUT_MS=60000 +# VEKTRA_LLM_CONTEXT_WINDOW= # Required for models not in litellm registry (e.g. local vLLM). Example: 32768 # VEKTRA_LLM_CONTEXT_ONLY_ENABLED=true # -------------------------------------------------------------------------- diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 49fcc3dc..11613cb3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ jobs: integration: name: Integration tests + NFR gates runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 @@ -72,8 +72,8 @@ jobs: VEKTRA_BOOTSTRAP_KEY: ci-bootstrap-key STARTUP_MS: ${{ steps.startup.outputs.startup_ms }} - - name: Query latency measurement (warn only) - if: always() && !cancelled() + - name: Query latency measurement (warn only, post-merge only) + if: always() && !cancelled() && github.event_name == 'push' continue-on-error: true run: | uv run pytest tests/nfr/test_performance.py \ diff --git a/.gitignore b/.gitignore index 6fdd6674..5c43335e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,9 @@ vektra-learn/static/ coverage/ .nyc_output/ +# Evaluation results (generated by eval harness) +tests/eval/results_*.jsonl + # Temporary files tmp/ temp/ diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index efcb317b..06201c9c 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -81,6 +81,216 @@ --- +### BUG-014: Conversation rows never created — persistent store silently discards all turns + +**Status**: planned | **Priority**: high | **Created**: 2026-03-24 +**Analysis**: `vektra-internal/stack/20260324-conversation-persistence-gap-analysis.md` +**Reopens**: BUG-010 (marked completed but acceptance criteria #2 not satisfied) + +**Context**: `PersistentConversationStore.create_conversation()` exists (conversation.py:123-148) but is never called. The learn endpoint generates a `conversation_id` UUID (BUG-010 fix) and passes it to the pipeline, but no `ConversationOrm` row is created in the database. When the pipeline calls `add_turn()`, it looks up the conversation row, finds nothing, logs `conversation_not_found` at warning level, and returns silently. Result: **zero turns are ever persisted**, multi-turn context is broken (get_history returns empty), and `GET /api/v1/conversations/{id}` returns 404. + +**Root cause**: `create_conversation()` requires `namespace_id` and `key_id`, which are available in the API layer but not in the pipeline. `QueryRequest` does not carry auth context. The implementation plan (20260301-core-conversations.md) stated the pipeline should call `create_conversation()`, but the pipeline was never given the required parameters. BUG-010 was closed after adding UUID generation without completing the DB creation step. + +**Additional finding (fixed)**: `PersistentConversationStore` was not registered in the `ProviderRegistry`, so `GET /api/v1/conversations/{id}` returned 503 even when the store was initialized. Fixed by adding `registry.register("conversation_store", "default", conversation_store)` in main.py. + +**Traceability**: REQ-049, ARCH-031, BUG-010, FEAT-004 (blocked by this) + +**Acceptance criteria**: +- [ ] `POST /api/v1/query`: when `conversation_id` is None, create `ConversationOrm` row with namespace_id and key_id, set ID on request +- [ ] `POST /api/v1/query`: when `conversation_id` is provided but row doesn't exist, create it (first-use from client-generated ID) +- [ ] `POST /api/v1/learn/query`: same behavior, deriving key_id from learn service context +- [ ] `add_turn()` successfully persists turns after conversation creation +- [ ] `get_history()` returns previous turns for multi-turn queries +- [ ] `GET /api/v1/conversations/{id}` returns conversation metadata +- [ ] Verified: `conversations` and `conversation_turns` tables populated after widget queries +- [ ] Pipeline code unchanged (no auth context leaking into QueryRequest) + +--- + +### BUG-015: ~~Reranker scores discarded after reranking — threshold applied to wrong scores~~ + +**Status**: completed | **Priority**: critical | **Created**: 2026-03-24 | **Completed**: 2026-03-25 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` + +**Context**: `RerankerService.rerank()` (reranker.py:54-60) reorders results but returns the original `SearchResult` objects with their cosine similarity scores intact. The flashrank/cross-encoder scores are used only for ordering, then discarded. The `_apply_retrieval_filter` (pipeline.py:100) then applies `VEKTRA_MIN_RELEVANCE_SCORE=0.3` to these original cosine scores, not the reranker scores. The reranker's relevance judgment and the threshold filter are effectively disconnected: a chunk the reranker ranks highly can still be filtered out if its original cosine similarity was below 0.3. + +**Root cause**: The reranker implementation (commit dcb0b54, 2026-03-05) was designed to only reorder, not to propagate scores. The test `test_rerank_returns_top_k_in_order` verifies ordering but not score propagation. ADR-0014 and the implementation plan do not specify score handling. + +**Traceability**: ARCH-056, ADR-0021, ADR-0014 + +**Acceptance criteria**: +- [ ] `RerankerService.rerank()` propagates reranker scores to `SearchResult.score` (or a new field) +- [ ] `_apply_retrieval_filter` uses the correct score (reranker if available, cosine if not) +- [ ] Score normalization: all reranker outputs normalized to 0-1 at the reranker boundary +- [ ] When reranker is disabled, behavior unchanged (cosine scores, same threshold) +- [ ] Test verifies score values after reranking, not just ordering +- [ ] Config `VEKTRA_MIN_RELEVANCE_SCORE` description updated to reflect it applies to the active scoring stage + +--- + +### BUG-016: ~~English-only reranker produces random scores on Italian content~~ + +**Status**: completed | **Priority**: high | **Created**: 2026-03-24 | **Completed**: 2026-03-25 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` +**Depends on**: BUG-015 (score propagation must work before reranker swap is meaningful) + +**Context**: The default reranker model `ms-marco-MiniLM-L-12-v2` (via flashrank) is trained exclusively on English MS MARCO data. Its English-uncased tokenizer splits Italian words into meaningless subword fragments. On Italian text, the reranker produces essentially random relevance scores, potentially degrading retrieval by reordering correctly-retrieved chunks into a worse order. + +The choice was made during the hybrid search design phase (2026-02-07) optimizing for deployment constraints (4MB, no GPU, 50ms). Multilingual support was delegated entirely to the embedding model. The RAG tuning campaign (600 queries, 6 combos) held the reranker constant and never evaluated alternatives. + +The system must support both Italian and English content/queries (and mixed), so the solution must be multilingual, not Italian-specific. + +**Alternatives evaluated** (see analysis doc for full comparison): + +| Model | Params | Multilingual | Quality | Memory | Config change only? | +|-------|--------|-------------|---------|--------|---------------------| +| bge-reranker-v2-m3 | 568M | 100+ langs, best Mr.TyDi | High | ~1.2GB GPU | Yes | +| jina-reranker-v2-base-multilingual | 278M | 100+ langs | Good | ~600MB GPU | Yes | +| ms-marco-MultiBERT-L-12 (flashrank) | ~150M | 100+ langs | Terrible (26.91 NDCG) | ~150MB CPU | Yes, but do not use | + +The `rerankers` library already supports cross-encoder backends. No code changes needed: +``` +VEKTRA_RERANK_PROVIDER=cross-encoder +VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 +``` + +**Traceability**: ARCH-036, ADR-0021 + +**Acceptance criteria**: +- [ ] Default reranker model changed to a multilingual model that supports Italian and English +- [ ] English-only model remains available via config for English-only deployments +- [ ] Performance validated on Italian and English test queries (requires eval harness, TECH-002) +- [ ] Documentation updated (configuration.md, .env.example) with multilingual model guidance +- [ ] Memory and latency impact documented + +--- + +### TECH-002: ~~RAG evaluation harness~~ + +**Status**: completed | **Priority**: high | **Created**: 2026-03-24 | **Completed**: 2026-03-25 +**Analysis**: `vektra-internal/stack/20260324-reranker-threshold-gap-analysis.md` + +**Context**: The RAG tuning campaign (2026-03-14-18) used manual testing across 600 queries with qualitative metrics. There is no automated, reproducible way to evaluate retrieval quality when components change (embedding model, reranker, threshold, chunk size). This gap allowed BUG-015 and BUG-016 to go undetected: component interactions were never tested systematically. + +**Scope**: +- 50-question curated dataset (Italian Constitution + at least one English-language source) +- Two-stage evaluation: retrieval-only (fast, no LLM) and end-to-end (RAGAS metrics) +- `make eval-retrieval` and `make eval-e2e` targets +- Paired comparison support (same questions, two configs) +- JSONL results storage for historical tracking + +**Metrics**: context recall (primary), context precision, faithfulness, answer relevancy. + +**Traceability**: ARCH-050 (three-tier evaluation strategy), ADR-0019 + +**Acceptance criteria**: +- [ ] Curated test dataset with ground truth contexts (JSON, versioned in repo) +- [ ] `make eval-retrieval` runs retrieval-only evaluation and outputs metrics +- [ ] `make eval-e2e` runs full pipeline evaluation with RAGAS +- [ ] Baseline results recorded for current Combo D configuration +- [ ] Documentation on how to add test questions and run evaluations + +--- + +### DEBT-010: ~~Recalibrate relevance threshold with empirical data~~ + +**Status**: completed | **Priority**: medium | **Created**: 2026-03-24 | **Completed**: 2026-03-25 +**Depends on**: BUG-015, BUG-016, TECH-002 + +**Context**: `VEKTRA_MIN_RELEVANCE_SCORE=0.3` was set per ADR-0021 for cosine similarity with `all-MiniLM-L6-v2` (Phase 1 embedding model). It was never varied in the tuning campaign and not recalibrated when: (a) the embedding model changed to `paraphrase-multilingual-MiniLM-L12-v2`, (b) hybrid search with RRF was enabled, (c) the reranker was added. Different scoring stages produce different distributions (cosine 0.2-0.8 cluster, flashrank bimodal near 0/1, RRF reciprocal). A single threshold cannot serve all correctly. + +Literature consensus: use top-k as primary control, low absolute threshold (0.15-0.2) as safety net. Consider hybrid filtering (absolute minimum + relative percentile). + +**Traceability**: ARCH-056, ADR-0021 + +**Acceptance criteria**: +- [ ] Threshold tested at 0.1, 0.15, 0.2, 0.25, 0.3 using eval harness (TECH-002) +- [ ] Optimal threshold determined for the active reranker + embedding model combination +- [ ] ADR-0021 updated with new calibration data +- [ ] Configuration supports different thresholds for reranked vs non-reranked modes (or hybrid filter) + +--- + +### BUG-017: ~~Context window fallback silently truncates prompt — most chunks discarded~~ + +**Status**: completed | **Priority**: high | **Created**: 2026-03-25 | **Completed**: 2026-03-25 + +**Context**: `_context_window_impl()` (pipeline.py:131-136) calls `litellm.get_max_tokens(model)` to determine the context window. For models not in litellm's registry (all local vLLM models like `openai//models/qwen35-27b`), it silently falls back to `_DEFAULT_CONTEXT_WINDOW = 4096`. With Qwen 3.5 27B (actual context: 32768), this causes the token budget allocator to use only ~900 tokens for chunks instead of ~18000. Result: 5 relevant chunks retrieved, but only 2 fit in the prompt, and the LLM produces an incomplete answer. + +**Discovered**: while analyzing conversation `5bf50682` in namespace `ita-100`. User asked "Quali tipi di liberta sono garantiti dalla Costituzione italiana? Elencali tutti con il relativo articolo". Pipeline retrieved 20 candidates, reranker selected 5 (scores 0.78-0.40), threshold kept all 5, but `build_prompt` only included 2 (`chunks_in_prompt: 2`). The answer listed 6 freedoms instead of ~12. + +**Root cause**: no logging or warning when `litellm.get_max_tokens()` fails and the fallback kicks in. The `_count_tokens_impl()` fallback (char/4) is similarly silent. + +**Proposed fix**: +1. Add `VEKTRA_LLM_CONTEXT_WINDOW` env var to LLMConfig (optional int, default None) +2. `_context_window_impl()`: if env var set, use it; else try litellm; on fallback, emit `structlog.warning("context_window_fallback", model=model, default=4096)` +3. `_count_tokens_impl()`: on fallback, emit `structlog.warning("token_count_fallback", model=model)` (once per model, not per call) +4. Same pattern for any other fallback/default in the pipeline + +**Traceability**: ARCH-055 (token budget allocation) + +**Acceptance criteria**: +- [ ] `VEKTRA_LLM_CONTEXT_WINDOW` env var added, used when set +- [ ] Warning logged when context window falls back to default +- [ ] Warning logged when token counting falls back to char/4 +- [ ] Fallback warnings emitted once per model (not per query) to avoid log spam +- [ ] Documentation updated (configuration.md, .env.example) + +--- + +### BUG-018: SSE streaming path does not return server-generated conversation_id + +**Status**: planned | **Priority**: medium | **Created**: 2026-03-25 + +**Context**: When a client calls `POST /api/v1/query` with `stream=true` and no `conversation_id`, the server creates a conversation row and passes the ID to the pipeline. However, the SSE event stream never emits this ID back to the client. The non-streaming path returns it in the JSON response (`conversation_id` field), but the streaming path has no equivalent. + +A client using SSE without generating its own `conversation_id` cannot discover which ID to use for subsequent turns, breaking multi-turn conversations. + +**Current impact**: low. The widget always generates `conversation_id` client-side, so production is unaffected. The bug affects direct API consumers using SSE without pre-generating an ID. + +**Proposed fix**: emit the `conversation_id` in the first SSE event (e.g. a `metadata` event before tokens start) or in the `done` event payload. + +**Traceability**: BUG-014 (conversation persistence), DEBT-011 (observability gaps) + +**Acceptance criteria**: +- [ ] SSE stream includes `conversation_id` in an event accessible before or after token streaming +- [ ] Client can extract the ID and use it for follow-up queries +- [ ] Non-streaming path behavior unchanged + +--- + +### DEBT-011: Conversation and query trace observability gaps + +**Status**: planned | **Priority**: medium | **Created**: 2026-03-25 +**Related**: BUG-018 (SSE conversation_id) + +**Context**: Diagnosing a conversation (`5bf50682`, namespace `ita-100`) revealed multiple observability gaps that make post-hoc analysis of query behavior difficult: + +1. **No API to read conversation turns**: `GET /api/v1/conversations/{id}` returns metadata (turn_count, namespace, timestamps) but no endpoint exposes the turns themselves. The only way to read them is via direct DB query with `pgp_sym_decrypt()`. + +2. **Query trace not persisted for streaming queries**: When `stream=true`, the trace is emitted via SSE but not saved to `query_traces` table. Non-streaming queries also don't persist traces unless the learn service stores them. The only evidence of a streamed query is a single `query_stream_complete` log line with response_id and duration, no step details. + +3. **response_id not stored in conversation_turns**: The `response_id` column exists but is never populated, making it impossible to correlate a conversation turn with its query trace. + +4. **No admin endpoint for query traces**: Traces can only be retrieved via the learn API (if persisted) or by grepping container logs (which don't survive restarts, see INFRA-005). + +**Proposed approach**: +1. Persist query traces for all queries (not just learn), controlled by a config flag (default: on in development, off in production) +2. Populate `response_id` in conversation_turns when saving a turn +3. Add `GET /api/v1/admin/conversations/{id}/turns` endpoint (admin scope) that decrypts and returns turns +4. Add `GET /api/v1/admin/traces/{response_id}` endpoint for trace lookup + +**Traceability**: ARCH-041 (QueryTrace), ADR-0011 (conversation encryption), ADR-0017 (audit/analytics separation) + +**Acceptance criteria**: +- [ ] Query traces persisted to DB for all pipelines (simple + advanced, sync + stream) +- [ ] `response_id` populated in `conversation_turns` on turn save +- [ ] Admin endpoint to read decrypted conversation turns +- [ ] Admin endpoint to retrieve query trace by response_id +- [ ] Trace persistence configurable (always in dev, opt-in in production) + +--- + ### DEBT-009: Debug logging for rewritten queries **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 @@ -1003,7 +1213,7 @@ The `AdvancedQueryPipeline` correctly calls `SparseEmbeddingProvider.embed_query ### BUG-010: ~~Learn query endpoint does not auto-create conversation on first query~~ -**Status**: completed | **Priority**: high | **Created**: 2026-03-16 | **Completed**: 2026-03-22 (v0.3.0) +**Status**: completed (partial — DB row creation missing, tracked as BUG-014) | **Priority**: high | **Created**: 2026-03-16 | **Completed**: 2026-03-22 (v0.3.0) **Origin**: Moodle integration testing (2026-03-16) **Context**: The learn query endpoint (`POST /api/v1/learn/query`) passes `conversation_id` through to the pipeline unchanged. When the widget sends the first query without a `conversation_id` (which is the normal flow), the pipeline receives `None`, skips history retrieval and turn saving, and returns `conversation_id: null`. The widget receives `null` and has nothing to save — so the second query also has no `conversation_id`. Result: **every query is a single-turn query with no conversation continuity**. @@ -1250,3 +1460,6 @@ The backend stores conversation turns in the database (used for multi-turn query | DEBT-004 (budget ordering) | Phase 2 | Pgvector returns score-desc in practice | | DEBT-005 (disconnect cancel) | Phase 2 | uvicorn handles it implicitly | | DEBT-008 (LRU plaintext cache) | Phase 2 | Replace lru_cache with TTLCache | +| BUG-017 (context window fallback) | Before next release | Silently truncates prompts with vLLM models | +| BUG-018 (SSE conversation_id) | Before next release | Streaming clients can't discover server-generated ID | +| DEBT-011 (conversation observability) | Post-Phase 2 | Cannot diagnose query behavior post-hoc | diff --git a/Dockerfile b/Dockerfile index c3351715..a0eb597c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,11 +70,10 @@ COPY vektra-app/src vektra-app/src # Include optional extras for Phase 2 vector store and sparse search support. # When INSTALL_UNSTRUCTURED=true, also install the Unstructured PDF extractor. ARG INSTALL_UNSTRUCTURED=false -RUN uv sync --frozen --no-editable --no-dev \ - && uv pip install 'qdrant-client==1.17.0' 'fastembed==0.7.4' \ - && if [ "$INSTALL_UNSTRUCTURED" = "true" ]; then \ - uv pip install 'torch' 'torchvision' --index-url https://download.pytorch.org/whl/cpu --reinstall \ - && uv pip install 'unstructured[pdf]>=0.15' 'pi-heif' 'sentence-transformers' --reinstall; \ +RUN if [ "$INSTALL_UNSTRUCTURED" = "true" ]; then \ + uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant --extra ocr; \ + else \ + uv sync --frozen --no-editable --no-dev --extra sparse --extra qdrant; \ fi # -------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 78768caf..7f0448d8 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ endif # Targets # -------------------------------------------------------------------------- -.PHONY: help up down health ingest query demo logs test lint reindex batch-ingest +.PHONY: help up down health ingest query demo logs test lint reindex batch-ingest eval-retrieval eval-e2e help: ## Show available targets @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*##/ { \ @@ -98,3 +98,9 @@ reindex: ## Create reindex job (skeleton): make reindex VER=2 [NS=default] batch-ingest: ## Batch ingest files: make batch-ingest DIR=path/to/docs [NS=default] $(if $(DIR),,$(error DIR is required. Usage: make batch-ingest DIR=path/to/docs)) @scripts/batch-ingest.sh "$(DIR)" "$(or $(NS),default)" + +eval-retrieval: ## Run retrieval-only evaluation (no LLM calls) + uv run python scripts/eval_retrieval.py $(EVAL_ARGS) + +eval-e2e: ## Run end-to-end RAG evaluation (with LLM calls) + uv run python scripts/eval_e2e.py $(EVAL_ARGS) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d91c99c9..e16cdbd4 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -27,21 +27,33 @@ If using a cloud provider, also set the corresponding API key (see [LLM provider | `VEKTRA_LLM_PROVIDER` | str | (required) | LLM model in litellm format | | `VEKTRA_LLM_API_KEY` | str | - | API key for the LLM provider. Not needed for Ollama. | | `VEKTRA_LLM_API_BASE` | str | - | Custom API base URL for OpenAI-compatible providers (e.g., vLLM, LMStudio) | +| `VEKTRA_LLM_EXTRA_BODY` | json | - | Extra JSON body passed to litellm (e.g. `{"chat_template_kwargs": {"enable_thinking": false}}` for vLLM thinking models) | | `VEKTRA_LLM_FALLBACK_MODEL` | str | - | Fallback model when the primary times out | | `VEKTRA_LLM_FALLBACK_TIMEOUT_MS` | int | `60000` | Milliseconds before switching to fallback model | +| `VEKTRA_LLM_CONTEXT_WINDOW` | int | - | Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup with 4096 fallback (warns on fallback) | | `VEKTRA_LLM_CONTEXT_ONLY_ENABLED` | bool | `true` | Return raw chunks without LLM synthesis when both models fail | | `VEKTRA_STARTUP_LLM_CHECK` | bool | `true` | Test LLM connectivity at startup (warning-only, not fatal) | ### LLM provider keys -Set one based on your `VEKTRA_LLM_PROVIDER`: +Use `VEKTRA_LLM_API_KEY` for any provider. It is passed directly to litellm and works with OpenAI, Anthropic, vLLM, and any OpenAI-compatible endpoint. -| Variable | Provider | -|----------|----------| -| `OPENAI_API_KEY` | OpenAI (`openai/*`) | -| `ANTHROPIC_API_KEY` | Anthropic (`anthropic/*`) | +Provider-specific environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) are also recognized by litellm as a fallback, but `VEKTRA_LLM_API_KEY` takes precedence when set. -These are standard provider environment variables recognized by litellm. `VEKTRA_LLM_API_KEY` can also be used as a generic alternative. +### Using a local vLLM instance + +```bash +VEKTRA_LLM_PROVIDER=openai//models/your-model-name +VEKTRA_LLM_API_KEY= +VEKTRA_LLM_API_BASE=http://:8000/v1 +VEKTRA_LLM_CONTEXT_WINDOW=32768 +# For models with thinking mode (e.g. Qwen3.5, DeepSeek-R1), disable it: +VEKTRA_LLM_EXTRA_BODY={"chat_template_kwargs": {"enable_thinking": false}} +``` + +In `.env` files (Docker Compose), JSON values do not need quoting. In shell, use single quotes: `VEKTRA_LLM_EXTRA_BODY='{"chat_template_kwargs": {"enable_thinking": false}}'`. + +The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35-27b`). ## Embedding @@ -67,7 +79,7 @@ These are standard provider environment variables recognized by litellm. `VEKTRA | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_QUERY_PIPELINE` | str | `advanced` | Pipeline implementation: `simple`, `advanced` | -| `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.3` | Minimum cosine similarity for chunk inclusion (0.0-1.0). Chunks below this threshold are filtered out. | +| `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.15` | Minimum relevance score for chunk inclusion (0.0-1.0). Safety net filter; top-k is the primary control. | | `VEKTRA_CHUNK_DEDUP_ENABLED` | bool | `true` | Deduplicate overlapping adjacent chunks from the same document | | `VEKTRA_RESPONSE_TOKEN_RESERVE` | int | `2048` | Tokens reserved for LLM response generation | | `VEKTRA_CONTEXT_CHUNK_RATIO` | float | `0.6` | Fraction of context window allocated to retrieved chunks (0.0-1.0) | @@ -85,8 +97,8 @@ These are standard provider environment variables recognized by litellm. `VEKTRA | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_RERANK_ENABLED` | bool | `true` | Enable cross-encoder reranking after retrieval | -| `VEKTRA_RERANK_PROVIDER` | str | `flashrank` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | -| `VEKTRA_RERANK_MODEL` | str | - | Provider-specific reranking model name | +| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | +| `VEKTRA_RERANK_MODEL` | str | `BAAI/bge-reranker-v2-m3` | Multilingual reranking model. For English-only lightweight deployments: provider=`flashrank`, model=`ms-marco-MiniLM-L-12-v2` | | `VEKTRA_RERANK_TOP_K` | int | `5` | Final top-k results after reranking | ## Ingestion diff --git a/scripts/eval_e2e.py b/scripts/eval_e2e.py new file mode 100644 index 00000000..9f4ad808 --- /dev/null +++ b/scripts/eval_e2e.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""End-to-end RAG evaluation harness (TECH-002). + +Reads a JSONL dataset, calls the Vektra query API for each question, +and computes answer quality metrics. Optionally uses RAGAS for +faithfulness and answer relevancy scoring. + +Usage: + python scripts/eval_e2e.py [OPTIONS] + +Requires VEKTRA_API_URL and VEKTRA_API_KEY environment variables. +For RAGAS metrics, install: uv pip install ragas datasets +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +DEFAULT_DATASET = "tests/eval/dataset.jsonl" +DEFAULT_OUTPUT = "tests/eval/results_e2e.jsonl" + + +@dataclass +class E2EResult: + id: str + question: str + category: str + language: str + answer: str | None + no_relevant_context: bool + num_sources: int + source_scores: list[float] = field(default_factory=list) + duration_ms: float = 0.0 + error: str | None = None + + +def load_dataset(path: str) -> list[dict]: + entries = [] + with open(path) as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError as e: + print(f"Warning: skipping line {line_num}: {e}", file=sys.stderr) + return entries + + +def evaluate_question(client: httpx.Client, entry: dict, top_k: int) -> E2EResult: + """Run a single RAG query and collect the response.""" + question_id = entry["id"] + question = entry["question"] + namespace = entry.get("namespace", "default") + category = entry.get("category", "unknown") + language = entry.get("language", "unknown") + + t0 = time.monotonic() + try: + resp = client.post( + "/api/v1/query", + json={ + "question": question, + "namespace": namespace, + "top_k": top_k, + }, + ) + resp.raise_for_status() + data = resp.json() + except Exception as e: + return E2EResult( + id=question_id, + question=question, + category=category, + language=language, + answer=None, + no_relevant_context=False, + num_sources=0, + duration_ms=(time.monotonic() - t0) * 1000, + error=str(e), + ) + + duration_ms = (time.monotonic() - t0) * 1000 + sources = data.get("sources", []) + + return E2EResult( + id=question_id, + question=question, + category=category, + language=language, + answer=data.get("answer"), + no_relevant_context=data.get("no_relevant_context", False), + num_sources=len(sources), + source_scores=[s["score"] for s in sources], + duration_ms=duration_ms, + ) + + +def print_summary(results: list[E2EResult], elapsed_s: float) -> None: + total = len(results) + errors = sum(1 for r in results if r.error) + evaluated = total - errors + + if evaluated == 0: + print("No questions evaluated successfully.") + return + + valid = [r for r in results if r.error is None] + answered = sum(1 for r in valid if r.answer is not None) + no_context = sum(1 for r in valid if r.no_relevant_context) + avg_sources = sum(r.num_sources for r in valid) / evaluated + + durations = sorted(r.duration_ms for r in valid) + p50 = durations[len(durations) // 2] + p95 = durations[int(len(durations) * 0.95)] + + print(f"\n{'=' * 60}") + print("End-to-end evaluation results") + print(f"{'=' * 60}") + print(f"Questions: {total} ({errors} errors)") + print(f"Duration: {elapsed_s:.1f}s total") + print("") + print(f"Answered: {answered}/{evaluated} ({answered / evaluated:.0%})") + print(f"No context: {no_context}/{evaluated}") + print(f"Avg sources: {avg_sources:.1f}") + print(f"Latency p50: {p50:.0f}ms") + print(f"Latency p95: {p95:.0f}ms") + + # Breakdown by category + categories = sorted(set(r.category for r in valid)) + if len(categories) > 1: + print("\nBy category:") + for cat in categories: + cat_results = [r for r in valid if r.category == cat] + cat_answered = sum(1 for r in cat_results if r.answer is not None) + print( + f" {cat:<15} answered={cat_answered}/{len(cat_results)} n={len(cat_results)}" + ) + + print(f"{'=' * 60}") + print("\nNote: RAGAS metrics (faithfulness, answer relevancy) require") + print("ground_truth_answer in the dataset and RAGAS installed.") + print("Install: uv pip install ragas datasets") + + +def save_results(results: list[E2EResult], output_path: str) -> None: + with open(output_path, "w") as f: + for r in results: + row = { + "id": r.id, + "question": r.question, + "category": r.category, + "language": r.language, + "answer": r.answer, + "no_relevant_context": r.no_relevant_context, + "num_sources": r.num_sources, + "source_scores": r.source_scores, + "duration_ms": round(r.duration_ms, 1), + } + if r.error: + row["error"] = r.error + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Vektra end-to-end evaluation harness") + parser.add_argument( + "--dataset", + default=DEFAULT_DATASET, + help=f"Path to JSONL dataset (default: {DEFAULT_DATASET})", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help=f"Path for JSONL results output (default: {DEFAULT_OUTPUT})", + ) + parser.add_argument( + "--top-k", type=int, default=5, help="Top-k for query (default: 5)" + ) + args = parser.parse_args() + + api_url = os.environ.get("VEKTRA_API_URL") + api_key = os.environ.get("VEKTRA_API_KEY") + if not api_key: + api_key_file = Path(".api-key") + if api_key_file.exists(): + api_key = api_key_file.read_text().strip() + + if not api_url or not api_key: + print( + "Error: VEKTRA_API_URL and VEKTRA_API_KEY must be set.", + file=sys.stderr, + ) + sys.exit(1) + + dataset = load_dataset(args.dataset) + if not dataset: + print(f"Error: no entries in {args.dataset}", file=sys.stderr) + sys.exit(1) + + print(f"Loaded {len(dataset)} questions from {args.dataset}") + print(f"API: {api_url} top_k={args.top_k}") + + client = httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=120.0, # LLM calls can be slow + ) + + results: list[E2EResult] = [] + t0 = time.monotonic() + for i, entry in enumerate(dataset): + result = evaluate_question(client, entry, args.top_k) + results.append(result) + status = "OK" if result.answer else ("ERR" if result.error else "NO_CTX") + answer_preview = (result.answer or "")[:60].replace("\n", " ") + print( + f" [{i + 1}/{len(dataset)}] {entry['id']} {status} {result.duration_ms:.0f}ms {answer_preview}" + ) + + elapsed = time.monotonic() - t0 + client.close() + + print_summary(results, elapsed) + save_results(results, args.output) + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py new file mode 100644 index 00000000..e5e35d98 --- /dev/null +++ b/scripts/eval_retrieval.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Retrieval-only evaluation harness (TECH-002). + +Reads a JSONL dataset, calls the Vektra search API for each question, +and computes retrieval quality metrics. No LLM calls are made. + +Usage: + python scripts/eval_retrieval.py [OPTIONS] + +Requires VEKTRA_API_URL and VEKTRA_API_KEY environment variables. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +DEFAULT_DATASET = "tests/eval/dataset.jsonl" +DEFAULT_OUTPUT = "tests/eval/results_retrieval.jsonl" + + +@dataclass +class QuestionResult: + id: str + question: str + category: str + language: str + hit: bool # at least one chunk matches expected keywords + reciprocal_rank: float # 1/rank of first relevant chunk (0 if no hit) + precision_at_k: float # relevant chunks / total chunks + num_retrieved: int + num_relevant: int + has_ground_truth: bool = True # False for entries without expected_keywords + scores: list[float] = field(default_factory=list) + error: str | None = None + + +def load_dataset(path: str) -> list[dict]: + entries = [] + with open(path) as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError as e: + print(f"Warning: skipping line {line_num}: {e}", file=sys.stderr) + return entries + + +def chunk_matches_keywords(text: str, keywords: list[str]) -> bool: + """Check if a chunk's text contains any of the expected keywords (case-insensitive).""" + text_lower = text.lower() + return any(kw.lower() in text_lower for kw in keywords) + + +def evaluate_question( + client: httpx.Client, + entry: dict, + top_k: int, + search_mode: str, + use_query: bool = False, +) -> QuestionResult: + """Run a single search/query and compute retrieval metrics. + + When use_query=True, calls /api/v1/query (includes reranker) and + evaluates on the returned sources. Otherwise calls /api/v1/search + (raw vector search, no reranker). + """ + question_id = entry["id"] + question = entry["question"] + keywords = entry.get("expected_keywords", []) + namespace = entry.get("namespace", "default") + category = entry.get("category", "unknown") + language = entry.get("language", "unknown") + + try: + if use_query: + resp = client.post( + "/api/v1/query", + json={ + "question": question, + "namespace": namespace, + "top_k": top_k, + }, + ) + else: + resp = client.post( + "/api/v1/search", + json={ + "query": question, + "namespace": namespace, + "top_k": top_k, + "search_mode": search_mode, + }, + ) + resp.raise_for_status() + data = resp.json() + except Exception as e: + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=False, + reciprocal_rank=0.0, + precision_at_k=0.0, + num_retrieved=0, + num_relevant=0, + has_ground_truth=False, + error=str(e), + ) + + # /api/v1/query returns "sources", /api/v1/search returns "results" + results = data.get("sources") or data.get("results") or [] + scores = [r["score"] for r in results] + + if not keywords: + # No ground truth: can only report retrieval count and scores + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=False, + reciprocal_rank=0.0, + precision_at_k=0.0, + num_retrieved=len(results), + num_relevant=0, + has_ground_truth=False, + scores=scores, + ) + + # Compute which chunks are relevant + # /api/v1/search uses "text_snippet", /api/v1/query uses "snippet" + relevant_mask = [ + chunk_matches_keywords(r.get("text_snippet") or r.get("snippet", ""), keywords) + for r in results + ] + num_relevant = sum(relevant_mask) + hit = num_relevant > 0 + + # MRR: reciprocal rank of first relevant chunk + reciprocal_rank = 0.0 + for i, is_relevant in enumerate(relevant_mask): + if is_relevant: + reciprocal_rank = 1.0 / (i + 1) + break + + # Precision@k + precision_at_k = num_relevant / len(results) if results else 0.0 + + return QuestionResult( + id=question_id, + question=question, + category=category, + language=language, + hit=hit, + reciprocal_rank=reciprocal_rank, + precision_at_k=precision_at_k, + num_retrieved=len(results), + num_relevant=num_relevant, + scores=scores, + ) + + +def print_summary(results: list[QuestionResult], elapsed_s: float) -> None: + total = len(results) + errors = sum(1 for r in results if r.error) + evaluated = total - errors + + if evaluated == 0: + print("No questions evaluated successfully.") + return + + valid = [r for r in results if r.error is None] + # Separate scored (have ground truth keywords) from unscored (adversarial) + scored = [r for r in valid if r.has_ground_truth] + unscored = [r for r in valid if not r.has_ground_truth] + + print(f"\n{'=' * 60}") + print("Retrieval evaluation results") + print(f"{'=' * 60}") + print( + f"Questions: {total} ({errors} errors, {len(unscored)} without ground truth)" + ) + print(f"Duration: {elapsed_s:.1f}s ({elapsed_s / total:.2f}s/query)") + + if scored: + hit_rate = sum(r.hit for r in scored) / len(scored) + mrr = sum(r.reciprocal_rank for r in scored) / len(scored) + avg_precision = sum(r.precision_at_k for r in scored) / len(scored) + avg_retrieved = sum(r.num_retrieved for r in scored) / len(scored) + avg_relevant = sum(r.num_relevant for r in scored) / len(scored) + + print(f"\nScored ({len(scored)} questions with ground truth):") + print(f" Hit rate: {hit_rate:.1%}") + print(f" MRR: {mrr:.4f}") + print(f" Avg precision: {avg_precision:.4f}") + print(f" Avg retrieved: {avg_retrieved:.1f}") + print(f" Avg relevant: {avg_relevant:.1f}") + else: + print("\nNo scored questions (all entries lack ground truth keywords).") + + # Breakdown by category (scored only) + if scored: + categories = sorted(set(r.category for r in scored)) + if len(categories) > 1: + print("\n By category:") + for cat in categories: + cat_results = [r for r in scored if r.category == cat] + cat_hit = sum(r.hit for r in cat_results) / len(cat_results) + cat_mrr = sum(r.reciprocal_rank for r in cat_results) / len(cat_results) + print( + f" {cat:<15} hit={cat_hit:.0%} mrr={cat_mrr:.4f} n={len(cat_results)}" + ) + + # Breakdown by language (scored only) + if scored: + languages = sorted(set(r.language for r in scored)) + if len(languages) > 1: + print("\n By language:") + for lang in languages: + lang_results = [r for r in scored if r.language == lang] + lang_hit = sum(r.hit for r in lang_results) / len(lang_results) + lang_mrr = sum(r.reciprocal_rank for r in lang_results) / len( + lang_results + ) + print( + f" {lang:<15} hit={lang_hit:.0%} mrr={lang_mrr:.4f} n={len(lang_results)}" + ) + + # Score distribution + all_scores = [s for r in valid for s in r.scores] + if all_scores: + all_scores.sort() + p25 = all_scores[len(all_scores) // 4] + p50 = all_scores[len(all_scores) // 2] + p75 = all_scores[3 * len(all_scores) // 4] + print( + f"\nScore distribution: min={all_scores[0]:.4f} p25={p25:.4f} p50={p50:.4f} p75={p75:.4f} max={all_scores[-1]:.4f}" + ) + + print(f"{'=' * 60}") + + +def save_results(results: list[QuestionResult], output_path: str) -> None: + with open(output_path, "w") as f: + for r in results: + row = { + "id": r.id, + "question": r.question, + "category": r.category, + "language": r.language, + "hit": r.hit, + "reciprocal_rank": r.reciprocal_rank, + "precision_at_k": r.precision_at_k, + "num_retrieved": r.num_retrieved, + "num_relevant": r.num_relevant, + "has_ground_truth": r.has_ground_truth, + "scores": r.scores, + } + if r.error: + row["error"] = r.error + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Vektra retrieval evaluation harness") + parser.add_argument( + "--dataset", + default=DEFAULT_DATASET, + help=f"Path to JSONL dataset (default: {DEFAULT_DATASET})", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help=f"Path for JSONL results output (default: {DEFAULT_OUTPUT})", + ) + parser.add_argument( + "--top-k", type=int, default=5, help="Top-k for search (default: 5)" + ) + parser.add_argument( + "--search-mode", + default="hybrid", + choices=["dense", "sparse", "hybrid"], + help="Search mode (default: hybrid)", + ) + parser.add_argument( + "--use-query", + action="store_true", + help="Use /api/v1/query (with reranker) instead of /api/v1/search", + ) + args = parser.parse_args() + + api_url = os.environ.get("VEKTRA_API_URL") + api_key = os.environ.get("VEKTRA_API_KEY") + if not api_key: + # Try .api-key file + api_key_file = Path(".api-key") + if api_key_file.exists(): + api_key = api_key_file.read_text().strip() + + if not api_url or not api_key: + print( + "Error: VEKTRA_API_URL and VEKTRA_API_KEY must be set.", + file=sys.stderr, + ) + sys.exit(1) + + dataset = load_dataset(args.dataset) + if not dataset: + print(f"Error: no entries in {args.dataset}", file=sys.stderr) + sys.exit(1) + + print(f"Loaded {len(dataset)} questions from {args.dataset}") + mode_info = ( + "query (with reranker)" if args.use_query else f"search mode={args.search_mode}" + ) + print(f"API: {api_url} top_k={args.top_k} {mode_info}") + + timeout = 120.0 if args.use_query else 30.0 + client = httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=timeout, + ) + + results: list[QuestionResult] = [] + t0 = time.monotonic() + for i, entry in enumerate(dataset): + result = evaluate_question( + client, entry, args.top_k, args.search_mode, use_query=args.use_query + ) + results.append(result) + status = "HIT" if result.hit else ("ERR" if result.error else "MISS") + print(f" [{i + 1}/{len(dataset)}] {entry['id']} {status}", end="") + if result.scores: + print(f" scores={[round(s, 3) for s in result.scores[:3]]}", end="") + print() + + elapsed = time.monotonic() - t0 + client.close() + + print_summary(results, elapsed) + save_results(results, args.output) + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/tests/eval/dataset.jsonl b/tests/eval/dataset.jsonl new file mode 100644 index 00000000..4ec6b8e1 --- /dev/null +++ b/tests/eval/dataset.jsonl @@ -0,0 +1,55 @@ +{"id": "IT-F-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione italiana?", "expected_keywords": ["diritti inviolabili", "formazioni sociali"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-02", "question": "Su cosa si fonda la Repubblica italiana?", "expected_keywords": ["fondata sul lavoro"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-03", "question": "Cosa stabilisce l'articolo 3 della Costituzione sull'uguaglianza?", "expected_keywords": ["pari dignita' sociale", "senza distinzione"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-04", "question": "A chi appartiene la sovranita' nella Repubblica italiana?", "expected_keywords": ["sovranita' appartiene al popolo"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-05", "question": "Cosa prevede la Costituzione italiana riguardo al diritto al lavoro?", "expected_keywords": ["diritto al lavoro", "condizioni che rendano effettivo"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-06", "question": "Come e' composta la bandiera della Repubblica italiana?", "expected_keywords": ["tricolore", "verde, bianco e rosso"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-07", "question": "Cosa tutela l'articolo 9 della Costituzione?", "expected_keywords": ["cultura", "ricerca scientifica", "paesaggio"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-08", "question": "Cosa stabilisce la Costituzione sulla liberta' personale?", "expected_keywords": ["liberta' personale", "inviolabile", "autorita' giudiziaria"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-09", "question": "Cosa prevede la Costituzione sulle minoranze linguistiche?", "expected_keywords": ["minoranze linguistiche", "tutela"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-10", "question": "Quali sono le garanzie per la liberta' di corrispondenza?", "expected_keywords": ["corrispondenza", "inviolabili", "autorita' giudiziaria"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-11", "question": "Cosa dice la Costituzione sul diritto di riunione?", "expected_keywords": ["riunirsi pacificamente", "senz'armi"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-12", "question": "Cosa prevede l'articolo 32 sulla salute?", "expected_keywords": ["salute", "fondamentale diritto", "cure gratuite"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-13", "question": "Cosa dice la Costituzione sull'obbligo scolastico?", "expected_keywords": ["istruzione inferiore", "obbligatoria e gratuita", "otto anni"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-14", "question": "Qual e' il rapporto tra Stato e Chiesa nella Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-F-15", "question": "Cosa prevede l'articolo 21 sulla liberta' di pensiero?", "expected_keywords": ["manifestare liberamente", "pensiero", "stampa"], "namespace": "default", "category": "factual", "language": "it"} +{"id": "IT-R-01", "question": "Qual e' il legame tra il principio di uguaglianza e il diritto al lavoro nella Costituzione?", "expected_keywords": ["uguaglianza", "lavoro"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-02", "question": "Come si conciliano autonomia locale e unita' nazionale nella Costituzione italiana?", "expected_keywords": ["autonomie locali", "una e indivisibile", "decentramento"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-03", "question": "In che modo la Costituzione bilancia liberta' di stampa e intervento giudiziario?", "expected_keywords": ["stampa", "sequestro", "autorita' giudiziaria"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-04", "question": "Quali limiti pone la Costituzione alla liberta' di circolazione?", "expected_keywords": ["circolare", "soggiornare", "sanita'", "sicurezza"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-05", "question": "Come protegge la Costituzione i cittadini capaci ma privi di mezzi economici nel percorso scolastico?", "expected_keywords": ["capaci e meritevoli", "borse di studio"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-06", "question": "In che modo la Costituzione italiana affronta il rapporto tra diritti individuali e solidarieta' sociale?", "expected_keywords": ["diritti inviolabili", "doveri inderogabili", "solidarieta'"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-07", "question": "Quali sono le condizioni per limitare la liberta' personale secondo la Costituzione?", "expected_keywords": ["atto motivato", "autorita' giudiziaria", "casi e modi previsti dalla legge"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-08", "question": "Come affronta la Costituzione la liberta' religiosa per le confessioni diverse dalla cattolica?", "expected_keywords": ["confessioni religiose", "egualmente libere", "intese"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-09", "question": "Qual e' il rapporto tra trattamento sanitario obbligatorio e rispetto della persona nella Costituzione?", "expected_keywords": ["trattamento sanitario", "disposizione di legge", "rispetto della persona umana"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "IT-R-10", "question": "Come la Costituzione tutela l'ambiente rispetto all'attivita' economica?", "expected_keywords": ["ambiente", "biodiversita'", "attivita' economica", "fini sociali e ambientali"], "namespace": "default", "category": "reasoning", "language": "it"} +{"id": "EN-F-01", "question": "What does Article 1 of the UDHR say about human beings?", "expected_keywords": ["born free and equal", "dignity and rights"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-02", "question": "What does the UDHR say about slavery?", "expected_keywords": ["slavery", "servitude", "prohibited"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-03", "question": "What rights does Article 19 of the UDHR protect?", "expected_keywords": ["freedom of opinion", "expression", "information"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-04", "question": "What does the UDHR say about education?", "expected_keywords": ["right to education", "free", "compulsory"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-F-05", "question": "What does Article 9 of the UDHR state about arrest?", "expected_keywords": ["arbitrary arrest", "detention", "exile"], "namespace": "default", "category": "factual", "language": "en"} +{"id": "EN-R-01", "question": "How does the UDHR connect the right to work with human dignity?", "expected_keywords": ["right to work", "remuneration", "dignity"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-02", "question": "What is the relationship between marriage and equality in the UDHR?", "expected_keywords": ["marry", "equal rights", "free and full consent"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-03", "question": "How does the UDHR protect against discrimination?", "expected_keywords": ["without distinction", "equal protection", "discrimination"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-04", "question": "What protections does the UDHR provide for motherhood and childhood?", "expected_keywords": ["motherhood", "childhood", "special care"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "EN-R-05", "question": "How does the UDHR balance individual rights with government authority?", "expected_keywords": ["will of the people", "elections", "universal", "suffrage"], "namespace": "default", "category": "reasoning", "language": "en"} +{"id": "MC-01", "question": "Come si confrontano le garanzie sulla liberta' personale nella Costituzione italiana e nella Dichiarazione Universale?", "expected_keywords": ["liberta' personale", "arbitrary arrest"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-02", "question": "Quali analogie esistono tra l'articolo 3 della Costituzione italiana e l'articolo 2 della UDHR sulla non discriminazione?", "expected_keywords": ["senza distinzione", "without distinction"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-03", "question": "How do both the Italian Constitution and the UDHR address freedom of religion?", "expected_keywords": ["fede religiosa", "freedom of thought"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-04", "question": "Quali somiglianze ci sono nel trattamento del diritto all'istruzione tra Costituzione italiana e UDHR?", "expected_keywords": ["istruzione", "education", "obbligatoria"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-05", "question": "How do both documents address the right to work and fair employment?", "expected_keywords": ["diritto al lavoro", "right to work"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-06", "question": "Quali differenze ci sono nella tutela della famiglia tra la Costituzione italiana e la UDHR?", "expected_keywords": ["famiglia", "matrimonio", "family"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-07", "question": "Come affrontano entrambi i documenti il diritto alla privacy e all'inviolabilita' del domicilio?", "expected_keywords": ["domicilio", "privacy", "correspondence"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-08", "question": "What similarities exist in how both documents treat freedom of expression and press?", "expected_keywords": ["pensiero", "expression", "stampa"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "MC-09", "question": "Come entrambi i documenti proteggono il diritto alla salute e a un tenore di vita adeguato?", "expected_keywords": ["salute", "health", "well-being"], "namespace": "default", "category": "multi-chunk", "language": "it"} +{"id": "MC-10", "question": "How do the Italian Constitution and UDHR compare on the presumption of innocence and fair trial?", "expected_keywords": ["presumed innocent", "autorita' giudiziaria"], "namespace": "default", "category": "multi-chunk", "language": "en"} +{"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-08", "question": "Cosa dice la Costituzione sul numero massimo di mandati presidenziali?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} +{"id": "ADV-09", "question": "What penalties does the UDHR prescribe for nations that violate human rights?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "en"} +{"id": "ADV-10", "question": "Come regolamenta la Costituzione italiana l'uso dei social media?", "expected_keywords": [], "namespace": "default", "category": "adversarial", "language": "it"} diff --git a/uv.lock b/uv.lock index 93a31c07..7928b3a8 100644 --- a/uv.lock +++ b/uv.lock @@ -5113,7 +5113,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.115" }, { name = "jinja2", specifier = ">=3.1" }, - { name = "litellm", specifier = ">=1.40" }, + { name = "litellm", specifier = ">=1.40,<1.82.7" }, { name = "presidio-analyzer", specifier = ">=2.2" }, { name = "presidio-anonymizer", specifier = ">=2.2" }, { name = "pydantic", specifier = ">=2,<3" }, diff --git a/vektra-app/src/vektra_app/main.py b/vektra-app/src/vektra_app/main.py index 2c13666c..6fd3e9f0 100644 --- a/vektra-app/src/vektra_app/main.py +++ b/vektra-app/src/vektra_app/main.py @@ -239,6 +239,8 @@ async def _step_5_register_providers( persistence="disabled", ) + registry.register("conversation_store", "default", conversation_store) + # --- Reranker (Phase 2, conditional) --- from vektra_core.reranker import create_reranker diff --git a/vektra-core/pyproject.toml b/vektra-core/pyproject.toml index fcf16d2b..a14a1d12 100644 --- a/vektra-core/pyproject.toml +++ b/vektra-core/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "vektra-shared", "fastapi>=0.115", "pydantic>=2,<3", - "litellm>=1.40", + "litellm>=1.40,<1.82.7", "jinja2>=3.1", "structlog>=24.0", "presidio-analyzer>=2.2", diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 2b753db0..cec6a638 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -103,7 +103,9 @@ def _count_tokens(self, text: str) -> int: return _count_tokens_impl(self._llm, self._llm_config.provider, text) def _context_window(self) -> int: - return _context_window_impl(self._llm_config.provider) + return _context_window_impl( + self._llm_config.provider, self._llm_config.context_window + ) async def _call_llm_with_fallback( self, diff --git a/vektra-core/src/vektra_core/api.py b/vektra-core/src/vektra_core/api.py index 471d9eae..eeb77e87 100644 --- a/vektra-core/src/vektra_core/api.py +++ b/vektra-core/src/vektra_core/api.py @@ -208,10 +208,34 @@ async def query( accept = request.headers.get("accept", "") use_stream = body.stream or "text/event-stream" in accept + # Ensure conversation row exists for persistent multi-turn (BUG-014). + # The API layer creates the row because it has namespace_id and key_id, + # which the pipeline does not (and should not) receive. + conversation_id = body.conversation_id + try: + conv_store = registry.get("conversation_store", "default") + if isinstance(conv_store, PersistentConversationStore): + if conversation_id is None: + conversation_id = await conv_store.create_conversation( + namespace_id=body.namespace, + key_id=_key.key_id, + ) + else: + # Client-provided ID: create row if it doesn't exist yet. + await conv_store.ensure_conversation( + conversation_id=conversation_id, + namespace_id=body.namespace, + key_id=_key.key_id, + ) + except ValueError: + pass # conversation store not registered (optional) + except Exception as exc: + log.warning("conversation_create_failed", error=str(exc)) + query_req = QueryRequest( question=body.question, namespace=body.namespace, - conversation_id=body.conversation_id, + conversation_id=conversation_id, top_k=body.top_k, stream=use_stream, ) diff --git a/vektra-core/src/vektra_core/conversation.py b/vektra-core/src/vektra_core/conversation.py index 33b67977..e23a2c28 100644 --- a/vektra-core/src/vektra_core/conversation.py +++ b/vektra-core/src/vektra_core/conversation.py @@ -125,17 +125,24 @@ async def create_conversation( namespace_id: str, key_id: UUID, title: str | None = None, + conversation_id: UUID | None = None, ) -> UUID: - """Create a new conversation row. Returns the conversation UUID.""" + """Create a new conversation row. Returns the conversation UUID. + + If *conversation_id* is provided, uses it as the primary key instead + of generating one server-side. This supports flows where the caller + has already allocated an ID (e.g. learn endpoint auto-generation). + """ async with self._session_factory() as session: + values: dict[str, Any] = { + "namespace_id": namespace_id, + "key_id": key_id, + "title": title, + } + if conversation_id is not None: + values["id"] = conversation_id stmt = ( - insert(ConversationOrm) - .values( - namespace_id=namespace_id, - key_id=key_id, - title=title, - ) - .returning(ConversationOrm.id) + insert(ConversationOrm).values(**values).returning(ConversationOrm.id) ) result = await session.execute(stmt) conversation_id = result.scalar_one() @@ -147,6 +154,32 @@ async def create_conversation( ) return conversation_id + async def ensure_conversation( + self, + conversation_id: UUID, + namespace_id: str, + key_id: UUID, + ) -> None: + """Create the conversation row if it does not already exist. + + Used when the caller provides a conversation_id (e.g. client-generated + or learn endpoint auto-generated) and the row may or may not exist yet. + """ + async with self._session_factory() as session: + from sqlalchemy.dialects.postgresql import insert as pg_insert + + stmt = ( + pg_insert(ConversationOrm) + .values( + id=conversation_id, + namespace_id=namespace_id, + key_id=key_id, + ) + .on_conflict_do_nothing(index_elements=["id"]) + ) + await session.execute(stmt) + await session.commit() + async def get_history(self, conversation_id: UUID) -> list[dict[str, str | None]]: """Return decrypted conversation history ordered by turn_number. diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index 4f1cdc92..540573fb 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -120,20 +120,50 @@ def _apply_retrieval_filter( # --------------------------------------------------------------------------- +_token_count_fallback_warned: set[str] = set() +_context_window_fallback_warned: set[str] = set() + + def _count_tokens_impl(llm: LLMProvider, model: str, text: str) -> int: """Count tokens using the LLM provider, with char/4 fallback.""" try: return llm.count_tokens(text, model) except Exception: + if model not in _token_count_fallback_warned: + _token_count_fallback_warned.add(model) + log.warning( + "token_count_fallback", + model=model, + method="char_div_4", + hint="Set VEKTRA_LLM_CONTEXT_WINDOW to ensure correct token budget allocation", + ) return max(1, len(text) // 4) -def _context_window_impl(model: str) -> int: - """Get context window size from litellm, with default fallback.""" +def _context_window_impl(model: str, configured_window: int | None = None) -> int: + """Get context window size, with fallback chain and warnings. + + Priority: configured_window (env var) > litellm lookup > default 4096. + """ + if configured_window is not None: + return configured_window + try: - return litellm.get_max_tokens(model) or _DEFAULT_CONTEXT_WINDOW + result = litellm.get_max_tokens(model) + if result: + return result except Exception: - return _DEFAULT_CONTEXT_WINDOW + pass + + if model not in _context_window_fallback_warned: + _context_window_fallback_warned.add(model) + log.warning( + "context_window_fallback", + model=model, + default=_DEFAULT_CONTEXT_WINDOW, + hint="Model not in litellm registry. Set VEKTRA_LLM_CONTEXT_WINDOW to the correct value", + ) + return _DEFAULT_CONTEXT_WINDOW async def _call_llm_with_fallback_impl( @@ -237,7 +267,9 @@ def _count_tokens(self, text: str) -> int: return _count_tokens_impl(self._llm, self._llm_config.provider, text) def _context_window(self) -> int: - return _context_window_impl(self._llm_config.provider) + return _context_window_impl( + self._llm_config.provider, self._llm_config.context_window + ) async def _call_llm_with_fallback( self, diff --git a/vektra-core/src/vektra_core/providers/litellm_provider.py b/vektra-core/src/vektra_core/providers/litellm_provider.py index 847e50af..b0ca180e 100644 --- a/vektra-core/src/vektra_core/providers/litellm_provider.py +++ b/vektra-core/src/vektra_core/providers/litellm_provider.py @@ -38,8 +38,12 @@ class LitellmProvider: def __init__(self, config: LLMConfig) -> None: self._config = config self._base_kwargs: dict[str, Any] = {} + if config.api_key: + self._base_kwargs["api_key"] = config.api_key if config.api_base: self._base_kwargs["api_base"] = config.api_base + if config.extra_body: + self._base_kwargs["extra_body"] = config.extra_body @property def model_name(self) -> str: diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 94bea399..9d7e9c70 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -2,11 +2,18 @@ Runs cross-encoder or flashrank reranking on vector search results. CPU-bound inference is offloaded to a thread via asyncio.to_thread(). + +Score propagation (BUG-015): reranker scores replace the original vector +similarity scores on SearchResult.score. The original score is preserved +in SearchResult.original_score for debugging. Scores are normalized to +[0, 1] via sigmoid when raw logits are detected (cross-encoder providers). """ from __future__ import annotations import asyncio +import dataclasses +import math import structlog @@ -23,6 +30,15 @@ } +def _sigmoid(x: float) -> float: + """Numerically stable sigmoid for cross-encoder logits.""" + if x >= 0: + z = math.exp(-x) + return 1.0 / (1.0 + z) + z = math.exp(x) + return z / (1.0 + z) + + class RerankerService: """Wraps the rerankers library for scoring and reordering search results.""" @@ -51,11 +67,24 @@ async def rerank( docs=docs, ) - # Build a mapping from original doc index to SearchResult + # Extract scores and detect whether normalization is needed. + # FlashRank produces sigmoid scores in [0, 1]; cross-encoder + # produces raw logits that can be negative or > 1. + top_items = ranked.results[:top_k] + raw_scores = [float(item.score) for item in top_items] + needs_sigmoid = any(s < 0.0 or s > 1.0 for s in raw_scores) + reranked: list[SearchResult] = [] - for item in ranked.results[:top_k]: + for item, raw_score in zip(top_items, raw_scores): original = results[item.doc_id] - reranked.append(original) + normalized = _sigmoid(raw_score) if needs_sigmoid else raw_score + reranked.append( + dataclasses.replace( + original, + score=normalized, + original_score=original.score, + ) + ) return reranked @@ -96,6 +125,6 @@ def _default_model_for_provider(provider: str) -> str: """Return a sensible default model for each provider.""" defaults = { "flashrank": "ms-marco-MiniLM-L-12-v2", - "cross-encoder": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "cross-encoder": "BAAI/bge-reranker-v2-m3", } return defaults.get(provider, provider) diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index d76f6f7e..22332a26 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -1,8 +1,10 @@ -You are a helpful assistant with access to a curated knowledge base. +You are a knowledgeable assistant. {% if namespace and namespace != "default" %}Namespace: {{ namespace }} {% endif %} -The user's message contains retrieved reference material inside tags. Each element is a passage from the knowledge base, not something the user wrote. Base your answer on these sources. -If the context does not contain enough information to answer the question, say so clearly. -Do not make up information that is not present in the context. -Do not mention or reference the context, sources, passages, or documents to the user. The user is not aware of the retrieval system. Answer naturally as if you know the information directly. -If a source passage is incomplete or cut off, use the available information without commenting on the truncation. +The user's message contains reference material inside tags. Each element is retrieved reference content, not something the user wrote. Use only this material to answer. If it does not fully cover the question, answer what you can and say you do not have information on the rest. + +Rules: +1. Sound like you simply know the answer. The user is not aware that reference material exists. Never mention, quote, or allude to texts, documents, sources, passages, or any material you were given. +2. Never offer to search, look up, or provide more information later. You cannot do that. +3. If a source is cut off, use what is available without commenting on the truncation. +4. Respond in the same language the user writes in. diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index 907aa6bc..6c6b1900 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -7,6 +7,10 @@ from vektra_core.pipeline import ( SimpleQueryPipeline, _apply_retrieval_filter, + _context_window_fallback_warned, + _context_window_impl, + _count_tokens_impl, + _token_count_fallback_warned, _token_overlap_ratio, ) from vektra_core.templates import TemplateRenderer @@ -466,3 +470,61 @@ async def _failing_stream(): assert "error" in types assert "trace" in types assert types[-1] == "done" + + +# --------------------------------------------------------------------------- +# _context_window_impl (BUG-017) +# --------------------------------------------------------------------------- + + +def test_context_window_uses_configured_value(): + """Configured context_window takes priority over litellm lookup.""" + result = _context_window_impl("nonexistent/model", configured_window=32768) + assert result == 32768 + + +def test_context_window_fallback_to_default(): + """Unknown model without configured window falls back to 4096 with warning.""" + _context_window_fallback_warned.discard("test/unknown-model-ctx") + result = _context_window_impl("test/unknown-model-ctx") + assert result == 4096 + assert "test/unknown-model-ctx" in _context_window_fallback_warned + + +def test_context_window_fallback_warns_once(capsys): + """Fallback warning is emitted on first call, silent on second.""" + _context_window_fallback_warned.discard("test/warn-once-model") + _context_window_impl("test/warn-once-model") + first = capsys.readouterr() + assert "context_window_fallback" in first.out + _context_window_impl("test/warn-once-model") + second = capsys.readouterr() + assert "context_window_fallback" not in second.out + + +# --------------------------------------------------------------------------- +# _count_tokens_impl fallback warning (BUG-017) +# --------------------------------------------------------------------------- + + +def test_count_tokens_fallback_warns(): + """Token count fallback emits warning on first use per model.""" + _token_count_fallback_warned.discard("test/token-fallback-model") + mock_llm = MagicMock() + mock_llm.count_tokens.side_effect = Exception("unsupported") + result = _count_tokens_impl(mock_llm, "test/token-fallback-model", "hello world") + assert result == max(1, len("hello world") // 4) + assert "test/token-fallback-model" in _token_count_fallback_warned + + +def test_count_tokens_fallback_warns_once(capsys): + """Token count fallback warning emitted on first call, silent on second.""" + _token_count_fallback_warned.discard("test/token-once-model") + mock_llm = MagicMock() + mock_llm.count_tokens.side_effect = Exception("unsupported") + _count_tokens_impl(mock_llm, "test/token-once-model", "first") + first = capsys.readouterr() + assert "token_count_fallback" in first.out + _count_tokens_impl(mock_llm, "test/token-once-model", "second") + second = capsys.readouterr() + assert "token_count_fallback" not in second.out diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index 742e221c..55072b54 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -7,6 +7,7 @@ from vektra_core.reranker import ( RerankerService, _default_model_for_provider, + _sigmoid, create_reranker, ) from vektra_shared.config import RerankConfig @@ -40,11 +41,11 @@ async def test_rerank_returns_top_k_in_order(): _make_result(0.9, "doc C"), ] - # Simulate rerankers output: list of items with doc_id attribute + # FlashRank-style scores (already 0-1) ranked_items = [ - SimpleNamespace(doc_id=2), # doc C first - SimpleNamespace(doc_id=0), # doc A second - SimpleNamespace(doc_id=1), # doc B third + SimpleNamespace(doc_id=2, score=0.95), # doc C first + SimpleNamespace(doc_id=0, score=0.80), # doc A second + SimpleNamespace(doc_id=1, score=0.10), # doc B third ] mock_ranker = MagicMock() mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) @@ -60,6 +61,87 @@ async def test_rerank_returns_top_k_in_order(): ) +async def test_rerank_propagates_flashrank_scores(): + """FlashRank scores (0-1) are propagated as-is (BUG-015).""" + results = [ + _make_result(0.5, "doc A"), + _make_result(0.3, "doc B"), + ] + + ranked_items = [ + SimpleNamespace(doc_id=1, score=0.85), + SimpleNamespace(doc_id=0, score=0.40), + ] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("query", results, top_k=2) + + # Reranker scores replace vector scores + assert reranked[0].score == 0.85 + assert reranked[1].score == 0.40 + # Original vector scores preserved + assert reranked[0].original_score == 0.3 + assert reranked[1].original_score == 0.5 + + +async def test_rerank_normalizes_cross_encoder_logits(): + """Cross-encoder logits (can be negative / > 1) are sigmoid-normalized (BUG-015).""" + results = [ + _make_result(0.6, "doc A"), + _make_result(0.4, "doc B"), + ] + + ranked_items = [ + SimpleNamespace(doc_id=0, score=3.5), # positive logit + SimpleNamespace(doc_id=1, score=-2.0), # negative logit + ] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("query", results, top_k=2) + + # Scores normalized via sigmoid + assert reranked[0].score == _sigmoid(3.5) + assert reranked[1].score == _sigmoid(-2.0) + # Normalized scores are in (0, 1) + assert 0.0 < reranked[0].score < 1.0 + assert 0.0 < reranked[1].score < 1.0 + # Original scores preserved + assert reranked[0].original_score == 0.6 + assert reranked[1].original_score == 0.4 + + +async def test_rerank_preserves_metadata(): + """Reranked results keep all original fields except score.""" + doc_id = uuid4() + results = [ + SearchResult( + chunk_id="chunk-1", + score=0.7, + text_snippet="hello", + document_id=doc_id, + document_version=3, + metadata={"page": 5}, + ), + ] + ranked_items = [SimpleNamespace(doc_id=0, score=0.99)] + mock_ranker = MagicMock() + mock_ranker.rank.return_value = SimpleNamespace(results=ranked_items) + + service = RerankerService(ranker=mock_ranker) + reranked = await service.rerank("q", results, top_k=1) + + assert reranked[0].chunk_id == "chunk-1" + assert reranked[0].document_id == doc_id + assert reranked[0].document_version == 3 + assert reranked[0].metadata == {"page": 5} + assert reranked[0].score == 0.99 + assert reranked[0].original_score == 0.7 + + # --------------------------------------------------------------------------- # create_reranker factory # --------------------------------------------------------------------------- @@ -87,11 +169,31 @@ def test_default_model_flashrank(): def test_default_model_cross_encoder(): - assert ( - _default_model_for_provider("cross-encoder") - == "cross-encoder/ms-marco-MiniLM-L-6-v2" - ) + assert _default_model_for_provider("cross-encoder") == "BAAI/bge-reranker-v2-m3" def test_default_model_unknown_provider(): assert _default_model_for_provider("custom") == "custom" + + +# --------------------------------------------------------------------------- +# _sigmoid +# --------------------------------------------------------------------------- + + +def test_sigmoid_zero(): + assert _sigmoid(0.0) == 0.5 + + +def test_sigmoid_large_positive(): + assert _sigmoid(10.0) > 0.999 + + +def test_sigmoid_large_negative(): + assert _sigmoid(-10.0) < 0.001 + + +def test_sigmoid_extreme_values_no_overflow(): + """Numerically stable sigmoid must not overflow on extreme logits.""" + assert _sigmoid(1000.0) == 1.0 + assert _sigmoid(-1000.0) == 0.0 diff --git a/vektra-learn/src/vektra_learn/api.py b/vektra-learn/src/vektra_learn/api.py index 88b8665d..2f85c678 100644 --- a/vektra-learn/src/vektra_learn/api.py +++ b/vektra-learn/src/vektra_learn/api.py @@ -16,6 +16,7 @@ import httpx import jwt +import structlog from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import StreamingResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer @@ -47,6 +48,9 @@ http_status_for, ) +# Sentinel key_id for learn-originated conversations (JWT auth has no API key). +_LEARN_SENTINEL_KEY_ID = UUID("00000000-0000-0000-0000-000000000000") + router = APIRouter(prefix="/api/v1/learn", tags=["learn"]) @@ -501,11 +505,30 @@ async def course_query( if req.conversation_id is None: req = req.model_copy(update={"conversation_id": uuid4()}) + # Ensure conversation row exists for persistent multi-turn (BUG-014). + # The learn endpoint uses JWT auth (no API key), so key_id is a sentinel. + registry = getattr(request.app.state, "registry", None) + if registry is not None: + try: + conv_store = registry.get("conversation_store", "default") + if ( + hasattr(conv_store, "ensure_conversation") + and req.conversation_id is not None + ): + await conv_store.ensure_conversation( + conversation_id=req.conversation_id, + namespace_id=namespace, + key_id=_LEARN_SENTINEL_KEY_ID, + ) + except ValueError: + pass # conversation store not registered + except Exception as exc: + structlog.get_logger(__name__).warning( + "conversation_create_failed", error=str(exc) + ) + # Build course-scoped query and delegate to pipeline query_req = build_course_query(req, namespace=namespace, course_id=course_id) - - # Get pipeline from ProviderRegistry - registry = getattr(request.app.state, "registry", None) if registry is None: err = ErrorResponse( category=ErrorCategory.TRANSIENT, diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index beb9f32c..a8d88cb4 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -9,6 +9,8 @@ from __future__ import annotations +from typing import Any + from pydantic import Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -31,6 +33,11 @@ class LLMConfig(BaseSettings): alias="VEKTRA_LLM_API_BASE", description="Custom API base URL for OpenAI-compatible providers (e.g., vLLM).", ) + extra_body: dict[str, Any] | None = Field( + None, + alias="VEKTRA_LLM_EXTRA_BODY", + description="Extra JSON body params passed to the LLM API (e.g., chat_template_kwargs for vLLM).", + ) fallback_model: str | None = Field( None, alias="VEKTRA_LLM_FALLBACK_MODEL", @@ -41,6 +48,12 @@ class LLMConfig(BaseSettings): alias="VEKTRA_LLM_FALLBACK_TIMEOUT_MS", description="Timeout in ms before switching to fallback model.", ) + context_window: int | None = Field( + None, + ge=1, + alias="VEKTRA_LLM_CONTEXT_WINDOW", + description="Context window size in tokens. Required for models not in litellm's registry (e.g. local vLLM). If unset, litellm lookup is attempted with a 4096-token fallback.", + ) context_only_enabled: bool = Field( True, alias="VEKTRA_LLM_CONTEXT_ONLY_ENABLED", @@ -143,12 +156,12 @@ class RerankConfig(BaseSettings): description="Enable reranking in AdvancedQueryPipeline.", ) provider: str = Field( - "flashrank", + "cross-encoder", alias="VEKTRA_RERANK_PROVIDER", description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere'.", ) model: str | None = Field( - None, + "BAAI/bge-reranker-v2-m3", alias="VEKTRA_RERANK_MODEL", description="Provider-specific reranking model name.", ) @@ -173,9 +186,9 @@ class QueryPipelineConfig(BaseSettings): description="QueryPipeline implementation: 'simple' (Phase 1), 'advanced' (Phase 2).", ) min_relevance_score: float = Field( - 0.3, + 0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE", - description="Minimum cosine similarity for chunk inclusion (ARCH-056).", + description="Minimum relevance score for chunk inclusion (ARCH-056). Safety net filter; top-k is the primary control.", ) chunk_dedup_enabled: bool = Field( True, @@ -415,6 +428,8 @@ class VektraSettings(BaseSettings): ) llm_api_key: str | None = Field(None, alias="VEKTRA_LLM_API_KEY") llm_api_base: str | None = Field(None, alias="VEKTRA_LLM_API_BASE") + llm_extra_body: dict[str, Any] | None = Field(None, alias="VEKTRA_LLM_EXTRA_BODY") + llm_context_window: int | None = Field(None, alias="VEKTRA_LLM_CONTEXT_WINDOW") llm_fallback_model: str | None = Field(None, alias="VEKTRA_LLM_FALLBACK_MODEL") llm_fallback_timeout_ms: int = Field(60000, alias="VEKTRA_LLM_FALLBACK_TIMEOUT_MS") llm_context_only_enabled: bool = Field( @@ -444,7 +459,7 @@ class VektraSettings(BaseSettings): # Query pipeline query_pipeline: str = Field("advanced", alias="VEKTRA_QUERY_PIPELINE") - min_relevance_score: float = Field(0.3, alias="VEKTRA_MIN_RELEVANCE_SCORE") + min_relevance_score: float = Field(0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE") chunk_dedup_enabled: bool = Field(True, alias="VEKTRA_CHUNK_DEDUP_ENABLED") response_token_reserve: int = Field(2048, alias="VEKTRA_RESPONSE_TOKEN_RESERVE") context_chunk_ratio: float = Field(0.6, alias="VEKTRA_CONTEXT_CHUNK_RATIO") @@ -523,6 +538,8 @@ def as_llm_config(self) -> LLMConfig: "VEKTRA_LLM_PROVIDER": self.llm_provider, "VEKTRA_LLM_API_KEY": self.llm_api_key, "VEKTRA_LLM_API_BASE": self.llm_api_base, + "VEKTRA_LLM_EXTRA_BODY": self.llm_extra_body, + "VEKTRA_LLM_CONTEXT_WINDOW": self.llm_context_window, "VEKTRA_LLM_FALLBACK_MODEL": self.llm_fallback_model, "VEKTRA_LLM_FALLBACK_TIMEOUT_MS": self.llm_fallback_timeout_ms, "VEKTRA_LLM_CONTEXT_ONLY_ENABLED": self.llm_context_only_enabled, diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index 797d65e3..0c056fea 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -156,6 +156,7 @@ class SearchResult: document_id: UUID document_version: int = 1 # from SourceDocument.version (REQ-056) metadata: dict[str, Any] = field(default_factory=dict) + original_score: float | None = None # pre-reranker score (BUG-015) @dataclass diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 15095e14..329e6f50 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -76,8 +76,8 @@ class TestRerankConfig: def test_defaults(self) -> None: cfg = RerankConfig() assert cfg.enabled is True - assert cfg.provider == "flashrank" - assert cfg.model is None + assert cfg.provider == "cross-encoder" + assert cfg.model == "BAAI/bge-reranker-v2-m3" assert cfg.top_k == 5 def test_env_var_override(self) -> None: @@ -147,7 +147,7 @@ def test_rewrite_and_rerank_defaults(self) -> None: assert cfg.rewrite.enabled is True assert cfg.rewrite.model is None assert cfg.rerank.enabled is True - assert cfg.rerank.provider == "flashrank" + assert cfg.rerank.provider == "cross-encoder" assert cfg.rerank.top_k == 5