From f43d014216a1b9c0c67567b64a60172b764019a9 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 08:48:41 +0000 Subject: [PATCH 01/31] test(shared): isolate unit tests from ambient VEKTRA_* env (DEBT-025) Importing litellm during pytest collection runs dotenv.load_dotenv(), leaking the developer's .env into os.environ and breaking 4 default-assertion tests in vektra-shared when the full suite runs (CI has no .env and never sees the difference). Add an autouse fixture in vektra-shared/tests/conftest.py that scrubs VEKTRA_* and external provider keys per-test via monkeypatch. Production settings loading is unchanged (no settings class uses env_file). Co-Authored-By: Claude Fable 5 --- vektra-shared/tests/conftest.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 vektra-shared/tests/conftest.py diff --git a/vektra-shared/tests/conftest.py b/vektra-shared/tests/conftest.py new file mode 100644 index 00000000..20fe69aa --- /dev/null +++ b/vektra-shared/tests/conftest.py @@ -0,0 +1,25 @@ +"""Shared fixtures for vektra-shared unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding the defaults asserted by test_config.py. CI has no .env and + never sees the difference. Tests that need a specific value still set it + explicitly via constructor kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) From ffb4acc232f435ce766c33827d81642336975088 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 08:48:41 +0000 Subject: [PATCH 02/31] docs(backlog): mark DEBT-025 completed; log fix in changelog Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 4 +++- CHANGELOG.md | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index ba87d95b..b3b3a75f 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -893,9 +893,11 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou ### DEBT-025: Isolate unit tests from the developer's local .env -**Status**: planned | **Priority**: low | **Created**: 2026-07-12 +**Status**: completed | **Priority**: low | **Created**: 2026-07-12 | **Completed**: 2026-07-12 **Origin**: discovered during the DEBT-024 sweep (PR #80): `make test` fails locally with 4 errors while CI is green. +**Resolution**: new `vektra-shared/tests/conftest.py` autouse fixture scrubs `VEKTRA_*` (plus `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`) from `os.environ` per-test via monkeypatch. Root cause confirmed: importing litellm during collection runs `dotenv.load_dotenv()`, leaking the repo `.env` into the process environment — running `vektra-shared/tests/test_config.py` alone passes, collecting it together with any litellm-importing package reproduces the 4 failures. Production settings loading untouched (no settings class uses `env_file`). + **Context**: 4 tests in `vektra-shared/tests/test_config.py` (`TestLLMConfig::test_defaults`, `TestQueryPipelineConfig::test_eval_mode_default_false`, `test_debug_log_queries_default_false`, `TestVektraSettings::test_defaults_with_required_only`) assert configuration defaults, but when the full suite runs from the workspace root the developer's `.env` leaks into `os.environ` (something imported during collection loads dotenv, e.g. litellm), so machine-specific values (eval_mode=true, custom port, LLM keys) override the defaults and the assertions fail. CI never sees this because runners have no `.env`. Current workaround: temporarily move `.env` away before `make test`. **Proposed approach**: a `vektra-shared/tests` (or workspace-level) autouse fixture that snapshots and scrubs `VEKTRA_*` variables from `os.environ` for the default-assertion tests, or `monkeypatch.delenv` on the specific vars. Alternatively point pydantic-settings at a nonexistent env file in tests via `_env_file=None`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 22332dfb..d7a1fbd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Convention (Keep a Changelog 1.1.0): +### Fixed + +- **tests**: unit tests are now hermetic against the developer's local `.env` (DEBT-025). Importing litellm during pytest collection loads `.env` into the process environment, which made 4 default-assertion tests in `vektra-shared` fail on dev machines while CI stayed green. An autouse fixture in `vektra-shared/tests/conftest.py` scrubs ambient `VEKTRA_*` variables; production settings loading is unchanged. + ## [0.5.1] - 2026-07-12 Security hardening: DEBT-024 dependency sweep, workflow permissions, admin login hardening. From 7b1bb6d7e8bdde9e0632f37b71f008865cda5c28 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 08:54:58 +0000 Subject: [PATCH 03/31] docs(s2s): add Sprint 3 RAG-quality implementation plan Plan grounded in codebase analysis (2026-07-12): corrects the FEAT-017 backlog premise (parent-child linkage is not persisted today), notes Qdrant must-only filters (FEAT-018 needs must_not), FEAT-020 namespace resolution as the pattern for FEAT-021 citations, and eval harness ground truths (no RAGAS, dataset all in namespace default). Co-Authored-By: Claude Fable 5 --- .s2s/plans/20260712-sprint3-rag-quality.md | 215 +++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 .s2s/plans/20260712-sprint3-rag-quality.md diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md new file mode 100644 index 00000000..2352febd --- /dev/null +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -0,0 +1,215 @@ +# Implementation Plan: Sprint 3 — RAG quality + +**ID**: 20260712-sprint3-rag-quality +**Status**: active +**Branch**: one branch/PR per item (`chore/debt-025-test-env-isolation`, `feat/feat-017-parent-chunk-expansion`, `feat/feat-021-namespace-citations`, FEAT-018 branch only if verification justifies it) +**Created**: 2026-07-12T08:39:46Z +**Updated**: 2026-07-12T09:00:00Z + +## Traceability + +**Source**: DEBT-025, FEAT-017, FEAT-018 (conditional on verification), FEAT-021 +**Source Type**: backlog + +Roadmap note: Sprint 3 in `vektra-internal/stack/20260321-implementation-roadmap-post-phase2.md` +listed FEAT-008, FEAT-013, FEAT-004, OQ-014. Deltas: FEAT-004 shipped in v0.5.0; +FEAT-013 is subsumed by FEAT-021 (citation titles/snippets); FEAT-008 stays out of +scope until it has a design (`/s2s:design`); OQ-014 not in this sprint. + +## References + +### Requirements +- REQ-055: response and citation traceability @.s2s/requirements.md (FEAT-021) +- REQ-051: no user text in traces (constrains multi-turn verification tooling; eval mode only) + +### Architecture +- ARCH-037: ChunkingStrategy (dual parent/child) @.s2s/architecture.md +- ARCH-055: token budget allocation @.s2s/architecture.md +- ARCH-056: retrieval quality controls @.s2s/architecture.md +- ARCH-054: composable Jinja2 templates @.s2s/architecture.md +- ARCH-047: namespace as first-class entity with metadata @.s2s/architecture.md +- ARCH-050: three-tier evaluation strategy @.s2s/architecture.md + +### Decisions +- ADR-0019: RAG evaluation strategy @.s2s/decisions/ADR-0019-rag-evaluation-strategy.md +- ADR-0021: retrieval quality controls @.s2s/decisions/ADR-0021-retrieval-quality-controls.md +- ADR-0023: conversational query rewriting @.s2s/decisions/ADR-0023-conversational-query-rewriting.md +- ADR-0020: prompt template architecture @.s2s/decisions/ADR-0020-prompt-template-architecture.md +- ADR-0025: learn chatbot widget @.s2s/decisions/ADR-0025-learn-chatbot-widget.md + +### Dependencies +- none (TECH-002 eval harness merged in 20260325-rag-retrieval-quality) + +## Overview + +Sprint objective: improve RAG answer quality **and measure it**. Every quality change +is bracketed by eval-harness runs (TECH-002) on the same dataset and corpus config +(Combo D: multilingual MiniLM embeddings, chunk 500/100, hybrid BM25+dense, reranker +bge-reranker-v2-m3 top_k=5, threshold 0.15, advanced pipeline). + +Order: (1) DEBT-025 warm-up so `make test` is green on dev machines, (2) baseline +eval, (3) FEAT-017 parent chunk expansion measured before/after, (4) FEAT-018 +verification-first — implement only if multi-turn tests show FEAT-020 did not +already mitigate it, (5) FEAT-021 per-namespace citations. + +## Design Notes + +Ground truths from codebase analysis (2026-07-12, develop @ ca25717) that correct +the backlog assumptions: + +- **Parent-child linkage is NOT stored** (FEAT-017 premise in backlog is wrong). + `DualStrategyChunking` builds the hierarchy in memory (`vektra-ingest/chunking.py:259-313`) + but `run_ingest` drops `chunk.parent_id` when building `ChunkEmbedding` + (`vektra-ingest/pipeline.py:400-418`); `ChunkEmbedding` has no parent_id field + (`vektra-shared/types.py:115-123`); Qdrant payload carries no parent_id + (`vektra-index/providers/qdrant.py:181-187`); `DocumentChunkOrm.parent_id` is always + NULL. Only `metadata.chunk_level` (parent|child) survives. FEAT-017 must plumb the + id through and **reingest** with `VEKTRA_CHUNKING_STRATEGY=dual`. +- **Parents ARE indexed and searchable today** when dual is enabled: `run_ingest` + embeds and upserts all chunks with no level filter; neither Qdrant `_build_filter` + nor pgvector excludes `chunk_level=parent`. The "search excludes parents" AC is + a real behavior change. +- **Qdrant mode keeps chunk text only in Qdrant**: with `VEKTRA_VECTOR_STORE_PROVIDER=qdrant` + (local setup), `document_chunks` in Postgres is empty; parent text must be fetched + via the vector store (Qdrant retrieve-by-id), not SQL. +- Parent size is hardcoded `chunk_size*3` at `vektra-ingest/pipeline.py:343-348` + (backlog's "3000 tokens" only holds for chunk_size=1000; with Combo D 500 it is 1500). +- **Qdrant filter builder is must-only** (`qdrant.py:376-431`); FEAT-018 chunk + exclusion needs `must_not` support. pgvector accepts `raw_filters` but silently + drops them (`pgvector.py:100-130`) — paper escape hatch. +- **Per-turn chunk ids are recoverable**: `conversation_turns.response_id` is populated + (BUG-013/DEBT-011) and `query_traces.chunks_retrieved` stores `[{chunk_id, score}]` + keyed by response_id — but only when trace storage is on (auto: dev on / prod off). +- **FEAT-020 pattern to clone for FEAT-021**: `resolve_grounding_mode` / + `resolve_show_sources` in `vektra-shared/namespace.py:19-82` read `namespaces.config` + JSONB; the API layer resolves and passes the value on `QueryRequest` + (`vektra-core/api.py:263-280`). Add `resolve_citations_enabled` the same way; + admin `PATCH /admin/namespaces/{id}/config` whitelist must accept the new key. +- **context.j2 uses ``**, not the `` shown + in the FEAT-021 backlog entry. `SearchResult.metadata` already carries `source_file` + and `page` from the Qdrant payload; `document_name` comes from `_fetch_document_names` + (currently applied to sources only, not to context rendering). Template edits change + `prompt_version` (trace comparability across the sprint). +- **Widget has no citation rendering**: `renderMarkdown` (bold/italic/code/links/lists + only) escapes literal `[1]`; inline citation markers hook into `renderInline` + (`vektra-learn/widget/src/markdown.js:136-153`). Bundle via esbuild to + `vektra-learn/static/vektra-chat.js`. +- **Eval harness**: HTTP clients only (`VEKTRA_API_URL` + `VEKTRA_API_KEY`; `.api-key` + file no longer exists locally — use `.env` value). e2e computes answerability + + latency; **RAGAS is not implemented** despite the TECH-002 AC text, and the dataset + has no `ground_truth_answer` fields. Retrieval metrics: hit rate, MRR, precision@k. + Dataset: 55 questions, all `namespace=default` (21 factual / 15 reasoning / + 10 multi-chunk / 9 adversarial; 37 IT / 18 EN). No multi-turn entries and no + `conversation_id` support in the scripts: FEAT-018 verification needs a small + multi-turn runner. +- **Local corpus state (2026-07-12)**: Qdrant collection `vektra` has 656 points but + only 12 in namespace `default` → the eval corpus must be (re)ingested before any + baseline. Current local chunking is `fixed` 500/100. + +## Tasks + +### 1. DEBT-025 — isolate unit tests from local .env (branch `chore/debt-025-test-env-isolation`) +- [x] Root cause confirmed: litellm import-time `load_dotenv()` during collection +- [x] Autouse fixture in `vektra-shared/tests/conftest.py` scrubs `VEKTRA_*` + external keys +- [x] `make test` green with populated `.env` (638 passed) + `make lint` green +- [x] Backlog entry updated (completed + resolution), changelog entry +- [ ] PR created and merged + +### 2. Baseline eval (no branch; results recorded, not committed as code) +- [ ] Eval corpus (re)ingested into namespace `default` (fixed 500/100, Combo D config) +- [ ] `make eval-retrieval` baseline recorded (hit rate, MRR, precision@k, per-category) +- [ ] `make eval-e2e` baseline recorded (grounded rate, no-ctx, latency p50/p95) +- [ ] Numbers logged in this plan (Notes) and in vektra-internal + +### 3. FEAT-017 — parent chunk expansion (branch `feat/feat-017-parent-chunk-expansion`) +- [ ] Propagate `parent_id` into `ChunkEmbedding` → Qdrant payload + pgvector column; + keep deterministic child/parent ids at store time +- [ ] Search excludes `chunk_level=parent` by default (both providers) +- [ ] `VEKTRA_PARENT_EXPANSION_ENABLED` (default false) in `QueryPipelineConfig` + + mirrored in `VektraSettings` +- [ ] Expansion step in AdvancedQueryPipeline: fetch parent text by id via vector + store, replace child text **before** token budgeting (ARCH-055); dedup children + of the same parent (child text is a substring of parent → existing 80% overlap + dedup interacts) +- [ ] Trace metadata records expansion (children expanded, parents fetched) +- [ ] Reingest eval corpus with `dual` strategy; measure: dual+exclusion without + expansion (≈ fixed baseline expected), then with expansion; record both +- [ ] Unit tests: store-time linkage, search filter, expansion logic, budget accounting + +### 4. FEAT-018 — verification first (no branch unless justified) +- [ ] Multi-turn scenario runner against `/api/v1/query` with `conversation_id` + (same-topic follow-up, topic switch, "give me others", negation) +- [ ] Evaluate with FEAT-020 grounding modes: does history use already mitigate + the "same chunks every turn" complaint? +- [ ] Decision recorded in backlog (implement with Qdrant `must_not` + turn-chunk + tracking, or close as mitigated) — implementation only if tests justify it + +### 5. FEAT-021 — per-namespace citations (branch `feat/feat-021-namespace-citations`) +- [ ] `resolve_citations_enabled` in `vektra_shared/namespace.py` (clone of FEAT-020 + pattern), default false; admin config PATCH whitelist extended +- [ ] `QueryRequest.citations_enabled` resolved at API layer, passed to renderer +- [ ] `system.j2` Rule 1 conditional (cite `[id]` inline when enabled) +- [ ] `context.j2`: add `title` attribute (document_name + page) when enabled +- [ ] Propagate `source_file`/`page`/`title` through to `SourceRef` (API response) +- [ ] Widget: render inline `[n]` markers as interactive references (tooltip/footnote + with source title + snippet); esbuild bundle rebuilt +- [ ] Default-off behavior byte-identical prompts (prompt_version changes documented) +- [ ] Unit tests: namespace resolution, both template branches, SourceRef fields + +## State & Data Lifecycle + +| State element | Created by | Updated by | Invalidated/Deleted by | +|---------------|-----------|------------|------------------------| +| Qdrant payload `parent_id` + `chunk_level` | ingest (FEAT-017) | reingest of document | document delete / collection reindex | +| `namespaces.config.citations_enabled` | admin config PATCH | admin config PATCH | key removal via PATCH null / namespace delete | +| `tests/eval/results_*.jsonl` | eval runs | overwritten per run | git history keeps baselines | +| conversation→chunk linkage (traces) | query pipeline (existing) | n/a (append-only) | analytics retention job | + +## Acceptance Criteria + +- [ ] Per-item ACs from `.s2s/BACKLOG.md` (DEBT-025 ✓, FEAT-017, FEAT-018-verification, FEAT-021) +- [ ] Baseline and post-change eval runs use the same dataset, corpus and config, + differing only in the variable under test; runs with per-question errors are + not accepted as baselines +- [ ] FEAT-018 go/no-go decision documented with test evidence in the backlog +- [ ] FEAT-021 default-off leaves current behavior unchanged (templates render + identically when disabled) +- [ ] `make lint` + `make test` green before every push + +## Testing Approach + +Unit tests accompany every code change (chunk linkage, search filters, expansion, +template conditionals, namespace resolution). Quality is measured with the TECH-002 +harness before/after each RAG-affecting change. Widget changes verified by rebuilding +the bundle and a manual smoke in the Moodle dev stack. + +### Test Infrastructure + +- **Required**: running stack (vektra + postgres + qdrant profile), eval corpus in + namespace `default`, reachable LLM (local vLLM via Tailscale IP), `VEKTRA_API_URL` + + `VEKTRA_API_KEY` env for the harness +- **Provided by**: `docker compose --profile qdrant up -d`; corpus reingest task in + items 2-3; existing `.env` (Combo D) +- **False-pass guard**: eval scripts mark per-question errors and print error counts; + a baseline with errors > 0 is rerun, not recorded. FEAT-018 verification failures + block implementation, not silently skip it. + +## Integration Notes + +- FEAT-021 touches the widget served by vektra-stack (`/static/learn/vektra-chat.js`); + the Moodle plugin embeds it unchanged — no vektra-moodle release needed unless new + `data-*` attributes are added (not planned: citations are namespace-driven). +- FEAT-017 requires a corpus reingest wherever dual strategy is enabled; local only, + no production deployments exist (no backward-compat constraints). +- Backlog drift found during analysis (FEAT-017 stored-linkage premise, FEAT-021 + template quote, TECH-002 RAGAS AC, DEBT-025 `_env_file` suggestion) is corrected + in the backlog as each item lands. + +## Notes + +- 2026-07-12: DEBT-025 completed on `chore/debt-025-test-env-isolation` + (fixture + docs, suite 638 passed / 3 skipped). Plan file rides the same PR. +- 2026-07-12: local stack recovered for eval: Qdrant container was down since ~May, + restarted from compose profile with existing volume (collection green, 656 points); + namespace `default` nearly empty → corpus reingest scheduled before baseline. +- Baseline numbers: TBD (item 2). From e2ab3368ac5963b06a48e6b17514d4c9e98cf8a0 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 09:07:13 +0000 Subject: [PATCH 04/31] fix(index): resolve search providers from the registry (BUG-021) /api/v1/search instantiated PgvectorProvider directly and read the sparse provider from a never-populated app.state attribute. In Qdrant deployments every search hit the empty document_chunks table (zero results with HTTP 200) and hybrid always fell back to dense. Resolve embedding, sparse embedding, and vector store from app.state.registry (the pipeline's contract); drop the per-request SentenceTransformers instantiation and the now-unused session dependency. Add endpoint tests. Found by the Sprint 3 baseline eval: make eval-retrieval returned zero results for all 55 questions against a healthy Qdrant stack. Co-Authored-By: Claude Fable 5 --- vektra-index/src/vektra_index/api.py | 38 +++---- vektra-index/tests/test_api_search.py | 157 ++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 vektra-index/tests/test_api_search.py diff --git a/vektra-index/src/vektra_index/api.py b/vektra-index/src/vektra_index/api.py index 5cf76738..3fff16b6 100644 --- a/vektra-index/src/vektra_index/api.py +++ b/vektra-index/src/vektra_index/api.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from vektra_shared.auth import ApiKeyInfo, require_scope -from vektra_shared.config import EmbeddingConfig, VectorStoreConfig +from vektra_shared.config import VectorStoreConfig from vektra_shared.db import get_session from vektra_shared.errors import ( ERR_INGEST_004, @@ -37,7 +37,6 @@ # Read from env once at import time (immutable for process lifetime) _VS_CONFIG = VectorStoreConfig() -_EMB_CONFIG = EmbeddingConfig() router = APIRouter(prefix="/api/v1", tags=["index"]) @@ -199,31 +198,26 @@ async def search( request: Request, body: SearchRequest, key: ApiKeyInfo = Depends(require_scope("query")), - session: AsyncSession = Depends(get_session), ) -> SearchResponse: """Semantic search over indexed chunks (no LLM synthesis). - Embeds the query string, then runs cosine similarity search with optional - JSONB metadata filtering. Returns ranked chunks. + Embeds the query string, then runs similarity search on the active + vector store provider with optional JSONB metadata filtering. + Returns ranked chunks. """ import logging - from vektra_index.providers.pgvector import PgvectorProvider - from vektra_index.providers.sentence_transformers import ( - SentenceTransformersProvider, - ) - _logger = logging.getLogger(__name__) + # Providers come from the registry: search must hit the active vector + # store (Qdrant when configured), not a hardcoded pgvector (BUG-021). + registry = request.app.state.registry + # Enforce namespace binding for scoped keys (H5) effective_ns = key.namespace_id or body.namespace - embedding_provider = SentenceTransformersProvider( - model_name=_EMB_CONFIG.embedding_model - ) - pgvector_provider = PgvectorProvider( - active_index_version=_VS_CONFIG.active_index_version - ) + embedding_provider = registry.get("embedding", "default") + vector_store = registry.get("vector_store", "default") # Embed the query (dense) - skip for SPARSE-only mode (NP24) dense_vector: list[float] = [] @@ -242,8 +236,8 @@ async def search( sparse_vector = None if body.search_mode in (SearchMode.SPARSE, SearchMode.HYBRID): - sparse_provider = getattr(request.app.state, "sparse_embedding_provider", None) - if sparse_provider is not None: + if registry.has("sparse_embedding", "default"): + sparse_provider = registry.get("sparse_embedding", "default") try: sparse_vector = await sparse_provider.embed_query(body.query) except Exception as exc: @@ -275,8 +269,7 @@ async def search( filters = body.filters # type: ignore[assignment] try: - results = await pgvector_provider.search( - session=session, + results = await vector_store.search( namespace=effective_ns, query_embedding=query_embedding, top_k=body.top_k, @@ -288,7 +281,10 @@ async def search( category=ErrorCategory.UPSTREAM, code=ERR_QUERY_004, message=f"Vector store read failed: {exc}", - remediation="Check PostgreSQL connectivity and pgvector extension status.", + remediation=( + "Check vector store connectivity and status. " + "Run GET /health for component status." + ), ) raise HTTPException( status_code=http_status_for(err), detail=err.to_envelope() diff --git a/vektra-index/tests/test_api_search.py b/vektra-index/tests/test_api_search.py new file mode 100644 index 00000000..a0e752fa --- /dev/null +++ b/vektra-index/tests/test_api_search.py @@ -0,0 +1,157 @@ +"""Unit tests for /api/v1/search provider resolution (BUG-021). + +The endpoint must resolve embedding, sparse embedding, and vector store from +the ProviderRegistry (same contract as the query pipeline), so that Qdrant +deployments search the active store instead of a hardcoded pgvector instance. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from vektra_shared.auth import ApiKeyInfo +from vektra_shared.registry import ProviderRegistry +from vektra_shared.types import SearchMode, SearchResult + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_key_store(scopes: list[str]) -> AsyncMock: + """Minimal mock key store that always accepts any token.""" + store = AsyncMock() + store.lookup_by_token = AsyncMock( + return_value=ApiKeyInfo(key_id=uuid4(), scopes=scopes) + ) + return store + + +def _make_vector_store(results: list[SearchResult]) -> AsyncMock: + store = AsyncMock() + store.search = AsyncMock(return_value=results) + return store + + +def _make_embedding_provider() -> AsyncMock: + provider = AsyncMock() + provider.embed_query = AsyncMock(return_value=[0.1, 0.2, 0.3]) + return provider + + +def _result(chunk_id: str = "chunk-1", score: float = 0.9) -> SearchResult: + return SearchResult( + chunk_id=chunk_id, + score=score, + text_snippet="Art. 2 diritti inviolabili", + document_id=uuid4(), + document_version=1, + metadata={}, + ) + + +def _make_app( + vector_store: AsyncMock, + sparse_provider: AsyncMock | None = None, +) -> FastAPI: + """Build a FastAPI test app with the index router and a stub registry.""" + from vektra_index.api import router + + app = FastAPI() + app.state.registry = ProviderRegistry() + app.state.registry.register("key_store", "default", _make_key_store(["query"])) + app.state.registry.register("embedding", "default", _make_embedding_provider()) + app.state.registry.register("vector_store", "default", vector_store) + if sparse_provider is not None: + app.state.registry.register("sparse_embedding", "default", sparse_provider) + app.include_router(router) + return app + + +async def _post_search(app: FastAPI, body: dict) -> tuple[int, dict]: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.post( + "/api/v1/search", + json=body, + headers={"Authorization": "Bearer test-token"}, + ) + return resp.status_code, resp.json() + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_search_uses_registry_vector_store(): + """Results must come from the registry's vector store, not pgvector.""" + vector_store = _make_vector_store([_result()]) + app = _make_app(vector_store) + + status, data = await _post_search( + app, + { + "query": "diritti", + "namespace": "default", + "top_k": 5, + "search_mode": "dense", + }, + ) + + assert status == 200 + assert data["total"] == 1 + assert data["results"][0]["chunk_id"] == "chunk-1" + + vector_store.search.assert_awaited_once() + kwargs = vector_store.search.await_args.kwargs + assert kwargs["namespace"] == "default" + assert kwargs["top_k"] == 5 + assert kwargs["search_mode"] == SearchMode.DENSE + assert kwargs["query_embedding"].dense == [0.1, 0.2, 0.3] + + +@pytest.mark.asyncio +async def test_search_hybrid_falls_back_to_dense_without_sparse(): + """HYBRID with no registered sparse provider degrades to DENSE.""" + vector_store = _make_vector_store([]) + app = _make_app(vector_store, sparse_provider=None) + + status, data = await _post_search( + app, + {"query": "q", "namespace": "default", "top_k": 3, "search_mode": "hybrid"}, + ) + + assert status == 200 + assert data["results"] == [] + kwargs = vector_store.search.await_args.kwargs + assert kwargs["search_mode"] == SearchMode.DENSE + assert kwargs["query_embedding"].sparse is None + + +@pytest.mark.asyncio +async def test_search_hybrid_uses_registered_sparse_provider(): + """HYBRID with a registered sparse provider keeps hybrid mode and sparse vector.""" + vector_store = _make_vector_store([_result()]) + sparse_vector = MagicMock(name="sparse_vector") + sparse_provider = AsyncMock() + sparse_provider.embed_query = AsyncMock(return_value=sparse_vector) + app = _make_app(vector_store, sparse_provider=sparse_provider) + + status, _ = await _post_search( + app, + {"query": "q", "namespace": "default", "top_k": 3, "search_mode": "hybrid"}, + ) + + assert status == 200 + sparse_provider.embed_query.assert_awaited_once_with("q") + kwargs = vector_store.search.await_args.kwargs + assert kwargs["search_mode"] == SearchMode.HYBRID + assert kwargs["query_embedding"].sparse is sparse_vector From 8ab1b7046f026beea800d5e3b7b96365c1016124 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 09:07:13 +0000 Subject: [PATCH 05/31] docs(backlog): record BUG-021 completed; changelog entry Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 21 +++++++++++++++++++++ CHANGELOG.md | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index ba87d95b..6e4e343d 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -923,6 +923,27 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou --- +### BUG-021: /api/v1/search hardwired to pgvector — empty results and no hybrid in Qdrant mode + +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-12 +**Origin**: Sprint 3 baseline eval (plan `20260712-sprint3-rag-quality`): `make eval-retrieval` returned zero results for all 55 questions against a healthy stack. + +**Context**: the search endpoint (`vektra-index/api.py`) instantiated `PgvectorProvider` directly and looked up the sparse provider in `request.app.state.sparse_embedding_provider`. Neither matches the app wiring: `main.py` registers providers in the ProviderRegistry (`vector_store`/`default` is overridden by Qdrant when `VEKTRA_VECTOR_STORE_PROVIDER=qdrant`; sparse under `sparse_embedding`/`default`; nothing is ever set on `app.state.sparse_embedding_provider`). Consequences in Qdrant deployments: (1) every search ran against the empty Postgres `document_chunks` table and returned `{"results": [], "total": 0}` with HTTP 200; (2) hybrid mode always fell back to dense with a `sparse_embedding_not_registered` warning even though FastEmbedBM25 was registered at startup. The RAG pipeline (`/api/v1/query`) was unaffected — it resolves providers from the registry — which is why the bug stayed invisible until the retrieval eval ran in qdrant mode. + +**Resolution**: the endpoint now resolves embedding, sparse embedding, and vector store from `request.app.state.registry` (same contract as the pipeline). The per-request `SentenceTransformersProvider` instantiation and the now-unused `session` dependency were removed. Unit tests added (`vektra-index/tests/test_api_search.py`): registry resolution, hybrid→dense fallback without sparse, hybrid with sparse. + +**Same family, not fixed here**: `POST /documents/{id}/chunks`, `DELETE /documents/{id}` and `GET /stats` still hardcode pgvector (the stats-vs-Qdrant mismatch was already a known issue). Track separately if needed. + +**Traceability**: ARCH-039 (ProviderRegistry), ARCH-051 (full-store contract), TECH-002 (eval harness) + +**Acceptance criteria**: +- [ ] `/api/v1/search` returns results in Qdrant mode (dense and hybrid) +- [ ] Hybrid uses the registered sparse provider (no spurious fallback) +- [ ] Unit tests pin registry-based provider resolution +- [ ] `make eval-retrieval` produces non-zero hit rate against the eval corpus + +--- + ### INFRA-005: Docker log persistence across container restarts **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22332dfb..fba0c9b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Convention (Keep a Changelog 1.1.0): +### Fixed + +- **index**: `/api/v1/search` now resolves the embedding, sparse-embedding, and vector-store providers from the ProviderRegistry instead of hardcoding pgvector and reading a never-populated `app.state` attribute (BUG-021). In Qdrant deployments the endpoint returned zero results (it searched the empty `document_chunks` table) and hybrid mode always fell back to dense; the RAG pipeline (`/api/v1/query`) was unaffected. Found by the Sprint 3 baseline `make eval-retrieval` run. + ## [0.5.1] - 2026-07-12 Security hardening: DEBT-024 dependency sweep, workflow permissions, admin login hardening. From 8f13b091c1965873f49e6607018e930b9b498646 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 09:20:56 +0000 Subject: [PATCH 06/31] docs(s2s): record sprint 3 baseline eval results in plan Baseline on the excerpt corpus (Combo D, qwen36-35b-a3b-fp8, strict): retrieval hit 100% / MRR 0.8957 / P@5 0.3261; e2e grounded 38/55 (69%), multi-chunk 3/10 is the bottleneck (chunks retrieved but cut by rerank+threshold before the prompt) - primary FEAT-017 target. Also logs the two blockers fixed to get here (BUG-021, stale vLLM model id). Co-Authored-By: Claude Fable 5 --- .s2s/plans/20260712-sprint3-rag-quality.md | 56 +++++++++++++++++----- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index 2352febd..d0ad9238 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -4,7 +4,7 @@ **Status**: active **Branch**: one branch/PR per item (`chore/debt-025-test-env-isolation`, `feat/feat-017-parent-chunk-expansion`, `feat/feat-021-namespace-citations`, FEAT-018 branch only if verification justifies it) **Created**: 2026-07-12T08:39:46Z -**Updated**: 2026-07-12T09:00:00Z +**Updated**: 2026-07-12T09:35:00Z ## Traceability @@ -102,9 +102,12 @@ the backlog assumptions: 10 multi-chunk / 9 adversarial; 37 IT / 18 EN). No multi-turn entries and no `conversation_id` support in the scripts: FEAT-018 verification needs a small multi-turn runner. -- **Local corpus state (2026-07-12)**: Qdrant collection `vektra` has 656 points but - only 12 in namespace `default` → the eval corpus must be (re)ingested before any - baseline. Current local chunking is `fixed` 500/100. +- **Local corpus state (2026-07-12)**: Qdrant collection `vektra` has 656 points; + the 12 in namespace `default` are the intact excerpt eval corpus from 20260325 + (`costituzione_italiana.md` v2 + `udhr_excerpts.md` v2 + `sample.pdf`). Current + local chunking is `fixed` 500/100. The full Costituzione PDF (97 chunks) was + deliberately removed from `default` back then; FEAT-017 will reintroduce a + larger corpus with `dual` chunking. ## Tasks @@ -116,10 +119,11 @@ the backlog assumptions: - [ ] PR created and merged ### 2. Baseline eval (no branch; results recorded, not committed as code) -- [ ] Eval corpus (re)ingested into namespace `default` (fixed 500/100, Combo D config) -- [ ] `make eval-retrieval` baseline recorded (hit rate, MRR, precision@k, per-category) -- [ ] `make eval-e2e` baseline recorded (grounded rate, no-ctx, latency p50/p95) -- [ ] Numbers logged in this plan (Notes) and in vektra-internal +- [x] Eval corpus verified intact in namespace `default` (excerpt corpus, 12 chunks — no reingest needed) +- [x] BUG-021 discovered and fixed (search endpoint hardwired to pgvector — see Notes; branch `fix/bug-021-search-registry-providers`) +- [x] `make eval-retrieval` baseline recorded (hit rate, MRR, precision@k, per-category) +- [x] `make eval-e2e` baseline recorded (grounded rate, no-ctx, latency p50/p95) +- [x] Numbers logged in this plan (Notes) and in vektra-internal (`stack/20260712-sprint3-baseline-eval.md`) ### 3. FEAT-017 — parent chunk expansion (branch `feat/feat-017-parent-chunk-expansion`) - [ ] Propagate `parent_id` into `ChunkEmbedding` → Qdrant payload + pgvector column; @@ -210,6 +214,36 @@ the bundle and a manual smoke in the Moodle dev stack. - 2026-07-12: DEBT-025 completed on `chore/debt-025-test-env-isolation` (fixture + docs, suite 638 passed / 3 skipped). Plan file rides the same PR. - 2026-07-12: local stack recovered for eval: Qdrant container was down since ~May, - restarted from compose profile with existing volume (collection green, 656 points); - namespace `default` nearly empty → corpus reingest scheduled before baseline. -- Baseline numbers: TBD (item 2). + restarted from compose profile with existing volume (collection green, 656 points). + The 12 points in namespace `default` turned out to be the intact excerpt eval + corpus (`costituzione_italiana.md` v2, 6 chunks + `udhr_excerpts.md` v2, 4 + + `sample.pdf`, 2) — no reingest needed for the baseline. +- 2026-07-12: baseline eval was blocked twice, both fixed: (a) **BUG-021** — + `/api/v1/search` was hardwired to pgvector and read the sparse provider from a + never-populated `app.state` attribute → zero results for all 55 questions in + qdrant mode, hybrid always degraded to dense. Fixed on + `fix/bug-021-search-registry-providers` (registry-based resolution + endpoint + tests). (b) stale vLLM model id in local `.env` (`qwen35-27b-fp8` after a + `vllm-switch`; server serves `qwen36-35b-a3b-fp8`) → every LLM call failed with + NotFoundError and the pipeline returned context-only answers (answer null). + `.env` updated, container recreated with the BUG-021 fix baked in. +- 2026-07-12: **baseline recorded** (excerpt corpus, Combo D, qwen36-35b-a3b-fp8, + grounding strict; full report in vektra-internal + `stack/20260712-sprint3-baseline-eval.md`): + - retrieval (`/api/v1/search` hybrid, top_k=5): hit rate 100% (46 scored), + MRR 0.8957, precision@5 0.3261; MRR factual 0.8810 / multi-chunk 0.8200 / + reasoning 0.9667; EN 0.8595 / IT 0.9115; RRF score p50 0.50. Hit rate + saturates on the tiny corpus — MRR/precision are the sensitive metrics. + - e2e (`/api/v1/query`, 55 questions, 0 errors, 178s): grounded 38/55 (69%), + no_context 17/55, answered-without-context 0, avg sources 1.3, latency + p50 3069ms / p95 4371ms. By category: factual 19/21, reasoning 12/15, + **multi-chunk 3/10**, adversarial 4/9 (adversarial no_ctx is largely the + desired refusal). + - reading: multi-chunk is the bottleneck — 7/10 end in no_context despite 100% + retrieval hit: chunks are found by search but cut by rerank+threshold before + the prompt (avg sources 1.3). Primary target for FEAT-017. Consider + reingesting the full Costituzione PDF (97 chunks) for a more discriminative + FEAT-017 comparison. + - March numbers (factual 90 / reasoning 80 / multi-chunk 10) are not comparable: + different pipeline (pre BUG-015/016/017, pre FEAT-020) and per-question + results were never versioned (gitignored file, since overwritten). From 66f592fde93024c327fdeee13f09a59e3924c242 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 16:05:53 +0000 Subject: [PATCH 07/31] docs(backlog): add TECH-005/TECH-006/FEAT-024/INFRA-007 from sprint 3 reviews TECH-005 eval suite expansion (collections, multi-turn runner, public regression slice); TECH-006 extractor/OCR bake-off + routing (GLM-OCR, PaddleOCR-VL, MinerU findings; tesseract-ita gap); FEAT-024 TEI remote embedding/reranker providers (the promised 'tei' option does not exist; includes the MiniLM 128-token truncation finding and the qdrant dense_dimensions latent bug); INFRA-007 GHCR image publishing for pull-based deployment updates. Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index ba87d95b..5e5c685c 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -299,6 +299,107 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. --- +### TECH-005: Eval suite expansion (collections, casistiche, public regression slice) + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: Sprint 3 baseline eval (plan `20260712-sprint3-rag-quality`) - the excerpt corpus saturates hit rate; the full-corpus variant (`tests/eval/dataset-full.jsonl`) exposed multi-chunk collapse. Operator wants broader casistiche coverage for performance, regression and tuning decisions. + +**Context**: today the harness has two datasets over two corpora (excerpts 12 chunks, `eval-full` 78 chunks) plus a documented "dirty extraction" trap (senato.it combined PDF: pdfplumber fuses words, IT hit rate 72% vs 88% clean - see `tests/eval/README.md`). Each new collection must cover a failure mode the existing ones cannot express; ground truth is keyword-based (chunking-independent), generated by an LLM reading the FULL documents (not the chunks, to avoid single-chunk bias), with human spot-check on a sample and adversarial entries authored against corpus gaps. + +**Collections to add, in value order for the e-learning vertical**: +1. Long structured "textbook" document (chapters/sections, context-dependent definitions) - the real FEAT-017 test bench; constitution articles are short and self-contained, parent expansion benefit is understated there. +2. Multi-document corpus with thematic distractors (e.g. Costituzione + UDHR + ECHR in one namespace) - stresses reranker discrimination and per-document citation correctness (FEAT-021). +3. Multi-turn conversational scenarios - harness currently never sends `conversation_id`; needed for FEAT-018 verification anyway. +4. Tables/exact-value questions (dates, numbers, formulas) - discriminates BM25 vs dense and table extraction. +5. Cross-lingual as a formal dataset category (EN queries on IT corpus and vice versa). +6. Dirty-extraction suite (senato PDF) as a separate ingest-robustness track with its own expectations. + +**Public regression slice**: add a small external benchmark for component regressions (embedding/reranker swaps), separate from the domain datasets: SQuAD-it subset for Italian (paragraph containing the answer = expected passage) and optionally BEIR/SciFact for English. Note: no public dataset provides "expected reranker scores" - they provide relevance judgments; assert ranking metrics (nDCG/MRR), never absolute score values (RRF scores are 1.0/0.5/0.33 by construction). + +**Generative metrics**: TECH-002's RAGAS AC was never implemented (no `ground_truth_answer` in datasets, no judge). Decide whether to add an LLM-judge stage (local vLLM as judge) with ground-truth answers on a subset. + +**Acceptance criteria**: +- [ ] At least the textbook collection + multi-turn runner implemented with documented ground-truth methodology +- [ ] Public regression slice runnable via `make eval-retrieval EVAL_ARGS=...` with recorded baseline +- [ ] `tests/eval/README.md` updated with collection matrix and when to use which +- [ ] Decision recorded on RAGAS/LLM-judge (implement or drop the TECH-002 AC) + +--- + +### TECH-006: Extractor/OCR bake-off and per-document routing design + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: operator request (system instance runs `unstructured` for comparison vs pdfplumber); OCR landscape research 2026-07-12 (sources in the entry). + +**Context**: ingestion quality gates retrieval quality (see the senato.it PDF case: fused words invalidate keyword ground truth). Current extractors: pdfplumber (default) and unstructured `strategy="auto"` (tesseract OCR fallback; **image ships English tesseract only - no `tesseract-ocr-ita`**). The DocumentExtractor Protocol routes per MIME type only; the pdfplumber-vs-unstructured choice is global config (`_build_extractor_registry`, `vektra-ingest/pipeline.py:78-112`). + +**Research findings (July 2026)**: +- "GLM OCR" is real: Zhipu GLM-OCR, 0.9B, MIT weights, #1 OmniDocBench v1.5 (94.62), very fast (MTP), vLLM-servable; Italian NOT in the official language list - must be validated empirically. +- PaddleOCR-VL 1.6 (Apache-2.0): highest composite (96.33 OmniDocBench v1.6), explicit Italian support (111+ langs), official NVIDIA Blackwell setup guide, vLLM/SGLang backends. +- MinerU 3.4 (Apache-based license): end-to-end PDF->MD with built-in scanned-PDF auto-detection, reading order, cross-page tables, 109-lang OCR incl. Italian; `http-client` backend can point at a remote vLLM. +- Docling (IBM, MIT): framework with pluggable OCR engines, selective OCR of bitmap regions, `force_full_page_ocr`; cleanest MIT-licensed lib companion/replacement for unstructured. +- unstructured supports swapping its OCR engine via `OCR_AGENT` env (tesseract -> paddle) without code changes. +- Marker/Surya/Chandra excluded (GPL / OpenRAIL-M commercial restrictions); olmOCR-2 excluded (English-focused); GOT-OCR2.0/Nougat superseded. +- Zero-install baseline available today: the active vLLM model (qwen36-35b-a3b, vision-capable) can be prompted as a page-image extractor. + +**Proposed approach**: +1. Quick wins: add `tesseract-ocr-ita` (build arg for language packs) to the INSTALL_UNSTRUCTURED image; evaluate `OCR_AGENT=paddle`. +2. Bake-off on a real IT+EN course-material corpus (lecture PDFs, scanned handouts, slides): pdfplumber vs unstructured(auto) vs MinerU vs PaddleOCR-VL vs GLM-OCR (Italian validation), scored with olmOCR-bench-style per-page checks + the TECH-005 dirty-extraction suite. +3. Routing design (follow-up FEAT): per-page/per-document signals (embedded text layer, image-coverage ratio, garble score, language) choosing extractor; MinerU/Docling already embed such routing if delegating at document level. + +**Traceability**: ARCH-037 (DocumentExtractor Protocol), ADR-0007, TECH-005 + +**Acceptance criteria**: +- [ ] Comparison report with per-tool scores on the shared corpus (IT + EN, native + scanned) +- [ ] Default extractor decision recorded (keep pdfplumber / switch / route) +- [ ] Routing feature filed as FEAT with concrete signals if the bake-off justifies it +- [ ] Image language packs fixed (tesseract-ita) regardless of outcome + +--- + +### FEAT-024: Remote embedding and reranker providers (TEI) + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: deployment modularity review 2026-07-12 - the host workstation already serves TEI instances (bge-m3, qwen3-embedding); Vektra cannot use them. + +**Context**: `VEKTRA_EMBEDDING_PROVIDER` documents a `tei` option (config.py:74) but **no TEI provider exists**: `main.py:129-137` unconditionally instantiates in-process `SentenceTransformersProvider`; the compose even ships a `tei` profile service nobody can talk to. The reranker likewise runs in-process only (`rerankers` lib; the `cohere` path never passes an api_key, so it is dead as wired - reranker.py:122). Consequences: every Vektra instance duplicates embedding/reranker compute in-container (CPU), and shared GPU/CPU inference services on the host cannot be reused. + +**Design**: +- `TEIEmbeddingProvider` implementing EmbeddingProvider over TEI HTTP (`POST /embed` or OpenAI-compatible `/v1/embeddings`), env: `VEKTRA_TEI_URL`, `VEKTRA_TEI_API_KEY`; `dimensions()` from TEI `GET /info`. Register on `VEKTRA_EMBEDDING_PROVIDER=tei`. +- `TEIRerankerProvider` over TEI `POST /rerank` (`{query, texts}` -> `[{index, score}]`, sigmoid scores; TEI serves bge-reranker-v2-m3 - one TEI instance per model). New `VEKTRA_RERANK_PROVIDER=tei` + endpoint/key env vars. Fix the cohere api_key pass-through in passing or remove the dead option. +- **Dimension plumbing**: `QdrantVectorStoreProvider` defaults `dense_dimensions=384` and main.py never passes it (qdrant.py:88, main.py:166-171) - a latent bug for any non-384 model; ensure collection creation uses the active provider's dimensions and document the reindex-on-model-change requirement (bge-m3 is 1024-dim). +- Note: TEI serves bge-m3 dense only (no sparse output for bge-m3); Vektra's BM25 sparse via fastembed stays client-side, hybrid keeps working. + +**Why it matters beyond dedup**: the current embedding model (paraphrase-multilingual-MiniLM-L12-v2) has **max_seq_length 128 tokens** - our 500-token chunks are silently truncated at embedding time (dense sees only the chunk head; BM25 sees the full text). bge-m3 (8192-token window, MIRACL dense nDCG@10 69.2 vs mE5-large 66.6; MiniLM sits 16-22 nDCG points below even mE5 on European-language retrieval per PL-MTEB) is the natural upgrade candidate, testable via TEI without fattening the container. + +**Traceability**: ADR-0013 (EmbeddingProvider Protocol), ARCH-035, ARCH-036, ADR-0021 + +**Acceptance criteria**: +- [ ] `VEKTRA_EMBEDDING_PROVIDER=tei` works end-to-end (ingest + query) against a TEI instance with api key +- [ ] Collection created with the provider's real dimensions; clear error on dimension mismatch with an existing collection +- [ ] `VEKTRA_RERANK_PROVIDER=tei` reranks via TEI /rerank with scores compatible with the threshold filter +- [ ] Embedding-model comparison (MiniLM in-process vs bge-m3 via TEI) run with the TECH-005/existing harness and recorded +- [ ] Docs: configuration.md + .env.example cover the new provider options + +--- + +### INFRA-007: Publish versioned container images on release (GHCR) + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: system-instance deployment 2026-07-12 - updates currently require a local `docker build` from a git checkout on every host. + +**Context**: no workflow publishes images (`release.yml` is a disabled placeholder, `if: false`, "Phase 1 releases are tagged manually"); the integration workflow builds only for its own tests. Deployments (e.g. the workstation rootful instance) must clone + build locally, which is slow and duplicates work per host. + +**Proposed approach**: GitHub Actions workflow on tag push (`v*`): build the image (both `INSTALL_UNSTRUCTURED=true` and `false` variants, e.g. tags `X.Y.Z` and `X.Y.Z-ocr`) and push to `ghcr.io/vektralabs/vektra`. Deployment update flow becomes `docker compose pull && docker compose up -d`. Consider enabling the semantic-release placeholder later; out of scope here. + +**Acceptance criteria**: +- [ ] Tag push publishes `ghcr.io/vektralabs/vektra:{version}` and `{version}-ocr` (multi-stage cache enabled) +- [ ] Image labels carry version + commit (OCI labels) +- [ ] README/deploy docs updated: pull-based deployment documented +- [ ] Existing tag flow unchanged (manual tagging still cuts the release) + +--- + ### BUG-013: QueryTrace not persisted to database **Status**: completed | **Priority**: high | **Created**: 2026-03-23 | **Completed**: 2026-04-04 | **PR**: #53 From 8155e047b3b6af511d525cc80a89c9f441636c95 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 16:15:24 +0000 Subject: [PATCH 08/31] fix(ingest): pin torchvision to the pytorch-cpu index for the ocr extra (BUG-022) uv resolved torchvision (transitive via unstructured-inference -> timm) from PyPI with CUDA-built wheels while torch is pinned to the CPU index, so any INSTALL_UNSTRUCTURED=true build crashed at the model pre-cache step with "operator torchvision::nms does not exist". Declare it in the ocr extra pinned to pytorch-cpu (torchvision 0.25.0+cpu; torch unchanged at 2.10.0, upper bound <0.26 keeps the relock surgical) and add a path-filtered CI workflow that builds the OCR variant so it cannot silently regress: regular CI never builds with the flag, which is why this was invisible since the extra was introduced. Co-Authored-By: Claude Fable 5 --- .github/workflows/docker-ocr-build.yml | 34 +++++++++ uv.lock | 96 +++++++++++++++++++------- vektra-ingest/pyproject.toml | 7 +- 3 files changed, 111 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/docker-ocr-build.yml diff --git a/.github/workflows/docker-ocr-build.yml b/.github/workflows/docker-ocr-build.yml new file mode 100644 index 00000000..9820929c --- /dev/null +++ b/.github/workflows/docker-ocr-build.yml @@ -0,0 +1,34 @@ +# Guard for the INSTALL_UNSTRUCTURED image variant (BUG-022). +# The OCR extra pulls torch-adjacent wheels whose index pinning can silently +# break the build; regular CI never builds with the flag, so this workflow +# does, but only when the inputs that can break it change. +name: docker-ocr-build + +on: + pull_request: + paths: + - "Dockerfile" + - "uv.lock" + - "pyproject.toml" + - "vektra-ingest/pyproject.toml" + - ".github/workflows/docker-ocr-build.yml" + push: + branches: [develop, main] + paths: + - "Dockerfile" + - "uv.lock" + - "pyproject.toml" + - "vektra-ingest/pyproject.toml" + - ".github/workflows/docker-ocr-build.yml" + +permissions: + contents: read + +jobs: + build-ocr-image: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - name: Build image with INSTALL_UNSTRUCTURED=true + run: docker build --build-arg INSTALL_UNSTRUCTURED=true -t vektra-ocr-ci . diff --git a/uv.lock b/uv.lock index 985f6c20..d0e7deed 100644 --- a/uv.lock +++ b/uv.lock @@ -4616,7 +4616,8 @@ dependencies = [ { name = "safetensors", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, { name = "torch", version = "2.10.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, - { name = "torchvision", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "torchvision", version = "0.25.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "torchvision", version = "0.25.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d7/2c/593109822fe735e637382aca6640c1102c19797f7791f1fd1dab2d6c3cb1/timm-1.0.25.tar.gz", hash = "sha256:47f59fc2754725735cc81bb83bcbfce5bec4ebd5d4bb9e69da57daa92fcfa768", size = 2414743, upload-time = "2026-02-23T16:49:00.137Z" } wheels = [ @@ -4766,34 +4767,76 @@ wheels = [ [[package]] name = "torchvision" version = "0.25.0" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'darwin'", +] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pillow", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "numpy", marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "pillow", marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, { name = "torch", version = "2.10.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, - { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/56/3a/6ea0d73f49a9bef38a1b3a92e8dd455cea58470985d25635beab93841748/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2abe430c90b1d5e552680037d68da4eb80a5852ebb1c811b2b89d299b10573b", size = 1874920, upload-time = "2026-01-21T16:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/51/f8/c0e1ef27c66e15406fece94930e7d6feee4cb6374bbc02d945a630d6426e/torchvision-0.25.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b75deafa2dfea3e2c2a525559b04783515e3463f6e830cb71de0fb7ea36fe233", size = 2344556, upload-time = "2026-01-21T16:27:40.125Z" }, - { url = "https://files.pythonhosted.org/packages/68/2f/f24b039169db474e8688f649377de082a965fbf85daf4e46c44412f1d15a/torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f25aa9e380865b11ea6e9d99d84df86b9cc959f1a007cd966fc6f1ab2ed0e248", size = 8072351, upload-time = "2026-01-21T16:27:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/16/8f650c2e288977cf0f8f85184b90ee56ed170a4919347fc74ee99286ed6f/torchvision-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9c55ae8d673ab493325d1267cbd285bb94d56f99626c00ac4644de32a59ede3", size = 4303059, upload-time = "2026-01-21T16:27:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5b/1562a04a6a5a4cf8cf40016a0cdeda91ede75d6962cff7f809a85ae966a5/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:24e11199e4d84ba9c5ee7825ebdf1cd37ce8deec225117f10243cae984ced3ec", size = 1874918, upload-time = "2026-01-21T16:27:39.02Z" }, - { url = "https://files.pythonhosted.org/packages/36/b1/3d6c42f62c272ce34fcce609bb8939bdf873dab5f1b798fd4e880255f129/torchvision-0.25.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f271136d2d2c0b7a24c5671795c6e4fd8da4e0ea98aeb1041f62bc04c4370ef", size = 2309106, upload-time = "2026-01-21T16:27:30.624Z" }, - { url = "https://files.pythonhosted.org/packages/c7/60/59bb9c8b67cce356daeed4cb96a717caa4f69c9822f72e223a0eae7a9bd9/torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:855c0dc6d37f462482da7531c6788518baedca1e0847f3df42a911713acdfe52", size = 8071522, upload-time = "2026-01-21T16:27:29.392Z" }, - { url = "https://files.pythonhosted.org/packages/32/a5/9a9b1de0720f884ea50dbf9acb22cbe5312e51d7b8c4ac6ba9b51efd9bba/torchvision-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:cef0196be31be421f6f462d1e9da1101be7332d91984caa6f8022e6c78a5877f", size = 4321911, upload-time = "2026-01-21T16:27:35.195Z" }, - { url = "https://files.pythonhosted.org/packages/52/99/dca81ed21ebaeff2b67cc9f815a20fdaa418b69f5f9ea4c6ed71721470db/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a8f8061284395ce31bcd460f2169013382ccf411148ceb2ee38e718e9860f5a7", size = 1896209, upload-time = "2026-01-21T16:27:32.159Z" }, - { url = "https://files.pythonhosted.org/packages/28/cc/2103149761fdb4eaed58a53e8437b2d716d48f05174fab1d9fcf1e2a2244/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:146d02c9876858420adf41f3189fe90e3d6a409cbfa65454c09f25fb33bf7266", size = 2310735, upload-time = "2026-01-21T16:27:22.327Z" }, - { url = "https://files.pythonhosted.org/packages/76/ad/f4c985ad52ddd3b22711c588501be1b330adaeaf6850317f66751711b78c/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c4d395cb2c4a2712f6eb93a34476cdf7aae74bb6ea2ea1917f858e96344b00aa", size = 8089557, upload-time = "2026-01-21T16:27:27.666Z" }, - { url = "https://files.pythonhosted.org/packages/63/cc/0ea68b5802e5e3c31f44b307e74947bad5a38cc655231d845534ed50ddb8/torchvision-0.25.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5e6b449e9fa7d642142c0e27c41e5a43b508d57ed8e79b7c0a0c28652da8678c", size = 4344260, upload-time = "2026-01-21T16:27:17.018Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1f/fa839532660e2602b7e704d65010787c5bb296258b44fa8b9c1cd6175e7d/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:620a236288d594dcec7634c754484542dc0a5c1b0e0b83a34bda5e91e9b7c3a1", size = 1896193, upload-time = "2026-01-21T16:27:24.785Z" }, - { url = "https://files.pythonhosted.org/packages/80/ed/d51889da7ceaf5ff7a0574fb28f9b6b223df19667265395891f81b364ab3/torchvision-0.25.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b5e7f50002a8145a98c5694a018e738c50e2972608310c7e88e1bd4c058f6ce", size = 2309331, upload-time = "2026-01-21T16:27:19.97Z" }, - { url = "https://files.pythonhosted.org/packages/90/a5/f93fcffaddd8f12f9e812256830ec9c9ca65abbf1bc369379f9c364d1ff4/torchvision-0.25.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:632db02300e83793812eee4f61ae6a2686dab10b4cfd628b620dc47747aa9d03", size = 8088713, upload-time = "2026-01-21T16:27:15.281Z" }, - { url = "https://files.pythonhosted.org/packages/1f/eb/d0096eed5690d962853213f2ee00d91478dfcb586b62dbbb449fb8abc3a6/torchvision-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1abd5ed030c708f5dbf4812ad5f6fbe9384b63c40d6bd79f8df41a4a759a917", size = 4325058, upload-time = "2026-01-21T16:27:26.165Z" }, - { url = "https://files.pythonhosted.org/packages/97/36/96374a4c7ab50dea9787ce987815614ccfe988a42e10ac1a2e3e5b60319a/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad9a8a5877782944d99186e4502a614770fe906626d76e9cd32446a0ac3075f2", size = 1896207, upload-time = "2026-01-21T16:27:23.383Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e2/7abb10a867db79b226b41da419b63b69c0bd5b82438c4a4ed50e084c552f/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:40a122c3cf4d14b651f095e0f672b688dde78632783fc5cd3d4d5e4f6a828563", size = 2310741, upload-time = "2026-01-21T16:27:18.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/e6/0927784e6ffc340b6676befde1c60260bd51641c9c574b9298d791a9cda4/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:846890161b825b38aa85fc37fb3ba5eea74e7091ff28bab378287111483b6443", size = 8089772, upload-time = "2026-01-21T16:27:14.048Z" }, - { url = "https://files.pythonhosted.org/packages/b6/37/e7ca4ec820d434c0f23f824eb29f0676a0c3e7a118f1514f5b949c3356da/torchvision-0.25.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f07f01d27375ad89d72aa2b3f2180f07da95dd9d2e4c758e015c0acb2da72977", size = 4425879, upload-time = "2026-01-21T16:27:12.579Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:724f212a58a0d0d758649ce288601056b5f46a01de545702f42bccc5b25cb0cc", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6c444119c6af8fa48b79c59f1529fc15a45a2bbf823cf85f851e7203d84f727f", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:80895a40faa5783ce19ed5a692a54062c7888a385490e866324355f8d59b2eb3", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a64ae543fe0a057a93954af94c239629723db196ed65090e6775f136726b22f8", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b7df71942e31b74a5161dea3452da35e5fb1d36c752aedf61f2dd3c7be35739a", upload-time = "2026-01-20T18:32:31Z" }, +] + +[[package]] +name = "torchvision" +version = "0.25.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "numpy", marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, + { name = "pillow", marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, + { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:727334e9a721cfc1ac296ce0bf9e69d9486821bfa5b1e75a8feb6f78041db481", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c1be164e93c68b2dbf460fd58975377c892dbcf3358fb72941709c3857351bba", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2d444009c0956669ada149f61ed78f257c1cc96d259efa6acf3929ca96ceb3f0", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fe54cbd5942cd0b26a90f1748f0d4421caf67be35c281c6c3b8573733a03d630", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:90eec299e1f82cfaf080ccb789df3838cb9a54b57e2ebe33852cd392c692de5c", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:783c8fc580bbfc159bff52f4f72cdd538e42b32956e70dffa42b940db114e151", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e985e12a9a232618e5a43476de5689e4b14989f5da6b93909c57afa57ec27012", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:813f0106eb3e268f3783da67b882458e544c6fb72f946e6ca64b5ed4e62c6a77", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:9212210f417888e6261c040495180f053084812cf873dedba9fc51ff4b24b2d3", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0c2d0da9bc011a0fde1d125af396a8fbe94d99becf9d313764f24ca7657a3448", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4d72a57a8f0b5146e26dac1fbfa2c905280cd04f5fcb23b9c56253506b683aeb", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:499eae1e535766391b6ee2d1e6e841239c20e2e6d88203a15b8f9f8d60a1f8bd", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7d47d544899fabac52ebe0d4812975608fd7ab79a3d7fb6383275eb667e33f53", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9511339b3b5eb75229e0b5041202e8aed9bef3b1de3a715b9fb319c9e97688fd", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:fb9f07f6a10f0ac24ac482ae68c6df99110b74a0d80a4c64fddc9753267d8815", upload-time = "2026-01-20T18:32:30Z" }, ] [[package]] @@ -5307,6 +5350,8 @@ dependencies = [ [package.optional-dependencies] ocr = [ + { name = "torchvision", version = "0.25.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "torchvision", version = "0.25.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, { name = "unstructured", extra = ["pdf"] }, ] @@ -5333,6 +5378,7 @@ requires-dist = [ { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0" }, { name = "structlog", specifier = ">=24.0" }, { name = "tiktoken", specifier = ">=0.7" }, + { name = "torchvision", marker = "extra == 'ocr'", specifier = ">=0.25,<0.26", index = "https://download.pytorch.org/whl/cpu" }, { name = "unstructured", extras = ["pdf"], marker = "extra == 'ocr'", specifier = ">=0.15" }, { name = "vektra-shared", editable = "vektra-shared" }, ] diff --git a/vektra-ingest/pyproject.toml b/vektra-ingest/pyproject.toml index 6269f730..cc153eac 100644 --- a/vektra-ingest/pyproject.toml +++ b/vektra-ingest/pyproject.toml @@ -21,7 +21,11 @@ dependencies = [ ] [project.optional-dependencies] -ocr = ["unstructured[pdf]>=0.15"] +# torchvision is a transitive dep (unstructured-inference -> timm) that must +# come from the pytorch-cpu index: the PyPI wheel is built against CUDA torch +# and fails at import against torch+cpu ("operator torchvision::nms does not +# exist", BUG-022). Declaring it here lets uv pin its index like torch. +ocr = ["unstructured[pdf]>=0.15", "torchvision>=0.25,<0.26"] [build-system] requires = ["hatchling"] @@ -38,6 +42,7 @@ markers = ["integration: integration tests requiring Docker (excluded from ci-un [tool.uv.sources] vektra-shared = { workspace = true } +torchvision = { index = "pytorch-cpu" } [dependency-groups] dev = [ From 7897d752027734d8e679cb2be19f0e7c6f4e4998 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 16:15:24 +0000 Subject: [PATCH 09/31] docs(backlog): record BUG-022 completed; changelog entry Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 18 ++++++++++++++++++ CHANGELOG.md | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index ba87d95b..aae28f42 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -907,6 +907,24 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou --- +### BUG-022: INSTALL_UNSTRUCTURED image build broken — torchvision resolved from PyPI against torch+cpu + +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-12 +**Origin**: first production build with `INSTALL_UNSTRUCTURED=true` (rootful system-instance deployment on the dev workstation, 2026-07-12). + +**Context**: the OCR image variant has never built successfully. `uv sync --extra ocr` resolves `torchvision` — a transitive dependency via `unstructured[pdf]` → unstructured-inference → timm — from PyPI, whose wheels are compiled against CUDA torch, while `torch` itself is pinned to the `pytorch-cpu` index (vektra-index `[tool.uv.sources]`). At image build the model pre-cache step (`Dockerfile:126`) fails importing sentence_transformers with `RuntimeError: operator torchvision::nms does not exist`. CI never builds with the flag, so the breakage stayed invisible since the extra was introduced. + +**Resolution**: declare `torchvision>=0.25,<0.26` in the `ocr` extra with `[tool.uv.sources] torchvision = { index = "pytorch-cpu" }` in `vektra-ingest/pyproject.toml` (same pattern as torch in vektra-index); relock — torchvision flips to `0.25.0+cpu` from the CPU index, torch stays at 2.10.0 (upper bound `<0.26` keeps the relock surgical). New path-filtered workflow `.github/workflows/docker-ocr-build.yml` builds the `INSTALL_UNSTRUCTURED=true` image whenever Dockerfile/uv.lock/ingest deps change, so the variant cannot silently regress again. + +**Note**: the OCR image ships English tesseract only (`tesseract-ocr-eng`); the missing Italian language pack is tracked as a TECH-006 quick win, out of scope here. + +**Acceptance criteria**: +- [ ] `docker build --build-arg INSTALL_UNSTRUCTURED=true .` succeeds from a clean cache +- [ ] `uv.lock` resolves torchvision from the pytorch-cpu registry +- [ ] CI builds the OCR variant on changes to Dockerfile / uv.lock / ingest deps + +--- + ### DEBT-026: Tune Prometheus instrumentation exclusions (health/docs endpoints) **Status**: planned | **Priority**: low | **Created**: 2026-07-12 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22332dfb..9d458291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Convention (Keep a Changelog 1.1.0): +### Fixed + +- **docker**: the `INSTALL_UNSTRUCTURED=true` image variant builds again (BUG-022). torchvision (transitive via unstructured-inference) resolved from PyPI with CUDA-built wheels while torch is pinned to the CPU index, crashing the build with `operator torchvision::nms does not exist`. It is now declared in the `ocr` extra and pinned to the pytorch-cpu index; a new path-filtered CI workflow builds the OCR variant so it cannot silently regress. + ## [0.5.1] - 2026-07-12 Security hardening: DEBT-024 dependency sweep, workflow permissions, admin login hardening. From d1fbe82704403a6bbf64a408d984731e89a7ec45 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 20:50:35 +0000 Subject: [PATCH 10/31] style(index): add return type annotations to search API tests Addresses review comments 3566046191, 3566046197, 3566046204 (gemini-code-assist): style guide rule 6 requires annotations on all public functions, including tests. Co-Authored-By: Claude Fable 5 --- vektra-index/tests/test_api_search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vektra-index/tests/test_api_search.py b/vektra-index/tests/test_api_search.py index a0e752fa..f494ac40 100644 --- a/vektra-index/tests/test_api_search.py +++ b/vektra-index/tests/test_api_search.py @@ -91,7 +91,7 @@ async def _post_search(app: FastAPI, body: dict) -> tuple[int, dict]: @pytest.mark.asyncio -async def test_search_uses_registry_vector_store(): +async def test_search_uses_registry_vector_store() -> None: """Results must come from the registry's vector store, not pgvector.""" vector_store = _make_vector_store([_result()]) app = _make_app(vector_store) @@ -119,7 +119,7 @@ async def test_search_uses_registry_vector_store(): @pytest.mark.asyncio -async def test_search_hybrid_falls_back_to_dense_without_sparse(): +async def test_search_hybrid_falls_back_to_dense_without_sparse() -> None: """HYBRID with no registered sparse provider degrades to DENSE.""" vector_store = _make_vector_store([]) app = _make_app(vector_store, sparse_provider=None) @@ -137,7 +137,7 @@ async def test_search_hybrid_falls_back_to_dense_without_sparse(): @pytest.mark.asyncio -async def test_search_hybrid_uses_registered_sparse_provider(): +async def test_search_hybrid_uses_registered_sparse_provider() -> None: """HYBRID with a registered sparse provider keeps hybrid mode and sparse vector.""" vector_store = _make_vector_store([_result()]) sparse_vector = MagicMock(name="sparse_vector") From cf58e08089dc31ce222861ec7918ee3a6c3ebd54 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 20:56:26 +0000 Subject: [PATCH 11/31] docs(backlog): fix TECH-006 traceability (ARCH-030/ARCH-042, not ARCH-037) ARCH-037 is the ChunkingStrategy Protocol; the DocumentExtractor-related decisions are ARCH-030 (pdfplumber extraction) and ARCH-042 (extractor dispatch). Addresses review comment 3566643596. Note: the reviewer's suggested ARCH-009 is the arq job payload design, also unrelated. Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 5e5c685c..a5454c77 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -347,7 +347,7 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. 2. Bake-off on a real IT+EN course-material corpus (lecture PDFs, scanned handouts, slides): pdfplumber vs unstructured(auto) vs MinerU vs PaddleOCR-VL vs GLM-OCR (Italian validation), scored with olmOCR-bench-style per-page checks + the TECH-005 dirty-extraction suite. 3. Routing design (follow-up FEAT): per-page/per-document signals (embedded text layer, image-coverage ratio, garble score, language) choosing extractor; MinerU/Docling already embed such routing if delegating at document level. -**Traceability**: ARCH-037 (DocumentExtractor Protocol), ADR-0007, TECH-005 +**Traceability**: ARCH-030 (pdfplumber extraction), ARCH-042 (extractor dispatch), ADR-0007, TECH-005 **Acceptance criteria**: - [ ] Comparison report with per-tool scores on the shared corpus (IT + EN, native + scanned) From f16c19abc0d6923306bbde7c3b66754253fd6273 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 20:56:40 +0000 Subject: [PATCH 12/31] ci(ingest): disable credential persistence in OCR build checkout The workflow only builds an image and never pushes to git, so the token actions/checkout stores in .git/config is unnecessary exposure (zizmor artipacked). Addresses review comment 3566659191. Co-Authored-By: Claude Fable 5 --- .github/workflows/docker-ocr-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-ocr-build.yml b/.github/workflows/docker-ocr-build.yml index 9820929c..9437aa8b 100644 --- a/.github/workflows/docker-ocr-build.yml +++ b/.github/workflows/docker-ocr-build.yml @@ -30,5 +30,7 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + with: + persist-credentials: false - name: Build image with INSTALL_UNSTRUCTURED=true run: docker build --build-arg INSTALL_UNSTRUCTURED=true -t vektra-ocr-ci . From 094c0f4b71c623be998472b3a48ef0da0c8b09d6 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 09:42:36 +0000 Subject: [PATCH 13/31] test(eval): add full-corpus dataset variant and harness README dataset-full.jsonl retargets the 55 questions to the eval-full namespace (clean full Italian Constitution from Wikisource, 72 chunks + official OHCHR UDHR PDF, 6 chunks). The excerpt corpus saturates hit rate at 100%; the full corpus is discriminative (hit 89.1%, MRR 0.806 at baseline). README documents corpus provenance, the senato.it PDF extraction trap, and how to run/extend the harness. Co-Authored-By: Claude Fable 5 --- tests/eval/README.md | 52 +++++++++++++++++++++++++++++++++ tests/eval/dataset-full.jsonl | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/eval/README.md create mode 100644 tests/eval/dataset-full.jsonl diff --git a/tests/eval/README.md b/tests/eval/README.md new file mode 100644 index 00000000..e09afe41 --- /dev/null +++ b/tests/eval/README.md @@ -0,0 +1,52 @@ +# RAG evaluation harness (TECH-002) + +Two-stage evaluation against a running Vektra stack. Both scripts are pure HTTP +clients: they require `VEKTRA_API_URL` and `VEKTRA_API_KEY` in the environment. + +```bash +make eval-retrieval # /api/v1/search (no LLM) - hit rate, MRR, precision@k +make eval-e2e # /api/v1/query (full pipeline + LLM) - grounded rate, latency +# extra args: +make eval-retrieval EVAL_ARGS="--dataset tests/eval/dataset-full.jsonl --output tests/eval/results_retrieval_full.jsonl" +``` + +Results are written as JSONL next to the datasets (`results_*.jsonl`, gitignored: +record aggregates in the active `.s2s/plans/` file and in vektra-internal). + +## Datasets + +| File | Namespace | Corpus | +|------|-----------|--------| +| `dataset.jsonl` | `default` | Excerpt corpus, 12 chunks: `costituzione_italiana.md` v2 (6), `udhr_excerpts.md` v2 (4), `sample.pdf` (2). Hit rate saturates here; useful for smoke/regression, not for tuning. | +| `dataset-full.jsonl` | `eval-full` | Full-document corpus, 78 chunks: clean full Italian Constitution (72) + official UDHR English PDF (6). Same 55 questions, discriminative metrics. | + +Both files share the same 55 questions (21 factual, 15 reasoning, 10 multi-chunk, +9 adversarial without ground truth; 37 IT / 18 EN). Entry shape: +`{id, question, expected_keywords, namespace, category, language}`. Relevance is +keyword-based (`expected_keywords`, diacritic-insensitive substring match), so it +is chunking-independent and survives reingestion. + +## Corpus provenance (eval-full) + +- `costituzione-full-clean.md`: full text of the Italian Constitution converted + from Wikisource (`https://it.wikisource.org/api/rest_v1/page/html/Costituzione_della_Repubblica_italiana`, + CC BY-SA), metadata header stripped. Kept outside the repo at + `/mnt/ai/datasets/vektra-eval/` on the dev machine. +- `udhr-en-ohchr.pdf`: official OHCHR English UDHR + (`https://www.ohchr.org/sites/default/files/UDHR/Documents/UDHR_Translations/eng.pdf`). +- Do NOT use the senato.it combined PDF (`costituzione.pdf`, 506 pages): its print + layout breaks pdfplumber extraction (fused words like + `COSTITUZIONEDELLAREPUBBLICAITALIANA`, preserved hyphenation) and invalidates + keyword ground truth. Measured impact: IT hit rate 72% garbled vs 88% clean on + the same questions. Worth keeping in mind as a future "dirty extraction" test + case, but never as a retrieval-quality corpus. + +To rebuild the corpus: download the two sources, then +`scripts/ingest.sh eval-full` for each. + +## Adding questions + +Append JSONL entries with a unique `id`, the target `namespace`, and 1-3 +`expected_keywords` that only appear in the passages that truly answer the +question. Adversarial entries (expected refusal) omit `expected_keywords` and are +reported separately (`has_ground_truth: false`). diff --git a/tests/eval/dataset-full.jsonl b/tests/eval/dataset-full.jsonl new file mode 100644 index 00000000..c486de7f --- /dev/null +++ b/tests/eval/dataset-full.jsonl @@ -0,0 +1,55 @@ +{"id": "IT-F-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione italiana?", "expected_keywords": ["diritti inviolabili", "formazioni sociali"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-02", "question": "Su cosa si fonda la Repubblica italiana?", "expected_keywords": ["fondata sul lavoro"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-03", "question": "Cosa stabilisce l'articolo 3 della Costituzione sull'uguaglianza?", "expected_keywords": ["pari dignita' sociale", "senza distinzione"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-04", "question": "A chi appartiene la sovranita' nella Repubblica italiana?", "expected_keywords": ["sovranita' appartiene al popolo"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-05", "question": "Cosa prevede la Costituzione italiana riguardo al diritto al lavoro?", "expected_keywords": ["diritto al lavoro", "condizioni che rendano effettivo"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-06", "question": "Come e' composta la bandiera della Repubblica italiana?", "expected_keywords": ["tricolore", "verde, bianco e rosso"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-07", "question": "Cosa tutela l'articolo 9 della Costituzione?", "expected_keywords": ["cultura", "ricerca scientifica", "paesaggio"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-08", "question": "Cosa stabilisce la Costituzione sulla liberta' personale?", "expected_keywords": ["liberta' personale", "inviolabile", "autorita' giudiziaria"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-09", "question": "Cosa prevede la Costituzione sulle minoranze linguistiche?", "expected_keywords": ["minoranze linguistiche", "tutela"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-10", "question": "Quali sono le garanzie per la liberta' di corrispondenza?", "expected_keywords": ["corrispondenza", "inviolabili", "autorita' giudiziaria"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-11", "question": "Cosa dice la Costituzione sul diritto di riunione?", "expected_keywords": ["riunirsi pacificamente", "senz'armi"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-12", "question": "Cosa prevede l'articolo 32 sulla salute?", "expected_keywords": ["salute", "fondamentale diritto", "cure gratuite"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-13", "question": "Cosa dice la Costituzione sull'obbligo scolastico?", "expected_keywords": ["istruzione inferiore", "obbligatoria e gratuita", "otto anni"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-14", "question": "Qual e' il rapporto tra Stato e Chiesa nella Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-15", "question": "Cosa prevede l'articolo 21 sulla liberta' di pensiero?", "expected_keywords": ["manifestare liberamente", "pensiero", "stampa"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-R-01", "question": "Qual e' il legame tra il principio di uguaglianza e il diritto al lavoro nella Costituzione?", "expected_keywords": ["uguaglianza", "lavoro"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-02", "question": "Come si conciliano autonomia locale e unita' nazionale nella Costituzione italiana?", "expected_keywords": ["autonomie locali", "una e indivisibile", "decentramento"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-03", "question": "In che modo la Costituzione bilancia liberta' di stampa e intervento giudiziario?", "expected_keywords": ["stampa", "sequestro", "autorita' giudiziaria"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-04", "question": "Quali limiti pone la Costituzione alla liberta' di circolazione?", "expected_keywords": ["circolare", "soggiornare", "sanita'", "sicurezza"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-05", "question": "Come protegge la Costituzione i cittadini capaci ma privi di mezzi economici nel percorso scolastico?", "expected_keywords": ["capaci e meritevoli", "borse di studio"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-06", "question": "In che modo la Costituzione italiana affronta il rapporto tra diritti individuali e solidarieta' sociale?", "expected_keywords": ["diritti inviolabili", "doveri inderogabili", "solidarieta'"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-07", "question": "Quali sono le condizioni per limitare la liberta' personale secondo la Costituzione?", "expected_keywords": ["atto motivato", "autorita' giudiziaria", "casi e modi previsti dalla legge"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-08", "question": "Come affronta la Costituzione la liberta' religiosa per le confessioni diverse dalla cattolica?", "expected_keywords": ["confessioni religiose", "egualmente libere", "intese"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-09", "question": "Qual e' il rapporto tra trattamento sanitario obbligatorio e rispetto della persona nella Costituzione?", "expected_keywords": ["trattamento sanitario", "disposizione di legge", "rispetto della persona umana"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-10", "question": "Come la Costituzione tutela l'ambiente rispetto all'attivita' economica?", "expected_keywords": ["ambiente", "biodiversita'", "attivita' economica", "fini sociali e ambientali"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "EN-F-01", "question": "What does Article 1 of the UDHR say about human beings?", "expected_keywords": ["born free and equal", "dignity and rights"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-02", "question": "What does the UDHR say about slavery?", "expected_keywords": ["slavery", "servitude", "prohibited"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-03", "question": "What rights does Article 19 of the UDHR protect?", "expected_keywords": ["freedom of opinion", "expression", "information"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-04", "question": "What does the UDHR say about education?", "expected_keywords": ["right to education", "free", "compulsory"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-05", "question": "What does Article 9 of the UDHR state about arrest?", "expected_keywords": ["arbitrary arrest", "detention", "exile"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-R-01", "question": "How does the UDHR connect the right to work with human dignity?", "expected_keywords": ["right to work", "remuneration", "dignity"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-02", "question": "What is the relationship between marriage and equality in the UDHR?", "expected_keywords": ["marry", "equal rights", "free and full consent"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-03", "question": "How does the UDHR protect against discrimination?", "expected_keywords": ["without distinction", "equal protection", "discrimination"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-04", "question": "What protections does the UDHR provide for motherhood and childhood?", "expected_keywords": ["motherhood", "childhood", "special care"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-05", "question": "How does the UDHR balance individual rights with government authority?", "expected_keywords": ["will of the people", "elections", "universal", "suffrage"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "MC-01", "question": "Come si confrontano le garanzie sulla liberta' personale nella Costituzione italiana e nella Dichiarazione Universale?", "expected_keywords": ["liberta' personale", "arbitrary arrest"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-02", "question": "Quali analogie esistono tra l'articolo 3 della Costituzione italiana e l'articolo 2 della UDHR sulla non discriminazione?", "expected_keywords": ["senza distinzione", "without distinction"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-03", "question": "How do both the Italian Constitution and the UDHR address freedom of religion?", "expected_keywords": ["fede religiosa", "freedom of thought"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-04", "question": "Quali somiglianze ci sono nel trattamento del diritto all'istruzione tra Costituzione italiana e UDHR?", "expected_keywords": ["istruzione", "education", "obbligatoria"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-05", "question": "How do both documents address the right to work and fair employment?", "expected_keywords": ["diritto al lavoro", "right to work"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-06", "question": "Quali differenze ci sono nella tutela della famiglia tra la Costituzione italiana e la UDHR?", "expected_keywords": ["famiglia", "matrimonio", "family"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-07", "question": "Come affrontano entrambi i documenti il diritto alla privacy e all'inviolabilita' del domicilio?", "expected_keywords": ["domicilio", "privacy", "correspondence"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-08", "question": "What similarities exist in how both documents treat freedom of expression and press?", "expected_keywords": ["pensiero", "expression", "stampa"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-09", "question": "Come entrambi i documenti proteggono il diritto alla salute e a un tenore di vita adeguato?", "expected_keywords": ["salute", "health", "well-being"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-10", "question": "How do the Italian Constitution and UDHR compare on the presumption of innocence and fair trial?", "expected_keywords": ["presumed innocent", "autorita' giudiziaria"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": ["pena di morte", "non e' ammessa"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-08", "question": "Cosa dice la Costituzione sul numero massimo di mandati presidenziali?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-09", "question": "What penalties does the UDHR prescribe for nations that violate human rights?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-10", "question": "Come regolamenta la Costituzione italiana l'uso dei social media?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} From 746ddc5a5229617f376abcbfc0e98f5e4471a8bf Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 21:14:44 +0000 Subject: [PATCH 14/31] feat(rag): plumb parent chunk linkage and expansion step (FEAT-017) Store time: DualStrategyChunking parents now carry their own hierarchy id; run_ingest remaps chunker-local ids to deterministic stored ids (uuid5(doc_id, position)) and sets ChunkEmbedding.parent_id on children. Qdrant persists parent_id as a top-level payload key; pgvector fills the existing (always-NULL until now) parent_id column and honors caller ids. Search time: both providers exclude chunk_level=parent from search results (parents are context material, not retrieval targets). New VectorStoreProvider.retrieve(namespace, chunk_ids) fetches chunks by id. Query time: AdvancedQueryPipeline step 6.5 (VEKTRA_PARENT_EXPANSION_ENABLED, default false) replaces child text with the parent chunk text after the retrieval filter and before token budgeting (ARCH-055); children of the same parent collapse into the highest-scored one. Trace records children_expanded, siblings_merged, parents_fetched. Co-Authored-By: Claude Fable 5 --- .env.example | 1 + docs/reference/configuration.md | 1 + .../src/vektra_core/advanced_pipeline.py | 71 ++++++++++++++++++ vektra-index/src/vektra_index/adapters.py | 11 +++ .../src/vektra_index/providers/pgvector.py | 73 ++++++++++++++++++- .../src/vektra_index/providers/qdrant.py | 46 +++++++++++- vektra-ingest/src/vektra_ingest/chunking.py | 5 +- vektra-ingest/src/vektra_ingest/pipeline.py | 14 ++++ vektra-ingest/tests/test_dual_chunking.py | 15 ++-- vektra-shared/src/vektra_shared/config.py | 8 ++ vektra-shared/src/vektra_shared/protocols.py | 6 ++ vektra-shared/src/vektra_shared/types.py | 2 + 12 files changed, 239 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 5cdbec2f..90a215fa 100644 --- a/.env.example +++ b/.env.example @@ -84,6 +84,7 @@ # VEKTRA_QUERY_PIPELINE=advanced # VEKTRA_MIN_RELEVANCE_SCORE=0.15 # VEKTRA_CHUNK_DEDUP_ENABLED=true +# VEKTRA_PARENT_EXPANSION_ENABLED=false # VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 # VEKTRA_PROMPT_TEMPLATES_DIR= diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c0753eed..48313c8f 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -81,6 +81,7 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | `VEKTRA_QUERY_PIPELINE` | str | `advanced` | Pipeline implementation: `simple`, `advanced` | | `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.15` | Minimum relevance score for chunk inclusion (0.0-1.0). Safety net filter; top-k is the primary control. | | `VEKTRA_CHUNK_DEDUP_ENABLED` | bool | `true` | Deduplicate overlapping adjacent chunks from the same document | +| `VEKTRA_PARENT_EXPANSION_ENABLED` | bool | `false` | Replace retrieved child chunks with their parent chunk text before prompt construction (advanced pipeline only). Requires documents ingested with `VEKTRA_CHUNKING_STRATEGY=dual`. | | `VEKTRA_RESPONSE_TOKEN_RESERVE` | int | `2048` | Tokens reserved for LLM response generation | | `VEKTRA_CONTEXT_CHUNK_RATIO` | float | `0.6` | Fraction of context window allocated to retrieved chunks (0.0-1.0) | | `VEKTRA_PROMPT_TEMPLATES_DIR` | str | - | Directory for custom Jinja2 prompt templates (`system.j2`, `context.j2`, `conversation.j2`). Uses built-in defaults if unset. | diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 68cee4aa..1a8eea16 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -8,6 +8,8 @@ 4. rerank - cross-encoder reranking (skip if not configured) 5. retrieval_filter - score threshold + overlap dedup (ARCH-056) 6. post_retrieval - SafeguardHook.post_retrieval() (DEBT-003) + 6.5 parent_expansion - replace child text with parent chunk text (FEAT-017, + skip unless VEKTRA_PARENT_EXPANSION_ENABLED) 7. build_prompt - token budget + Jinja2 rendering (ARCH-055) 8. llm_call - LLM with graceful degradation (ARCH-043) 9. pre_response - SafeguardHook.pre_response() (ARCH-049) @@ -16,6 +18,7 @@ from __future__ import annotations import asyncio +import dataclasses import hashlib import time from collections.abc import AsyncGenerator, AsyncIterator @@ -99,6 +102,7 @@ def __init__( self._rewrite_enabled = pipeline_config.rewrite.enabled self._eval_mode = pipeline_config.eval_mode self._debug_log_queries = pipeline_config.debug_log_queries + self._parent_expansion = pipeline_config.parent_expansion_enabled # -- Helpers (delegating to shared module-level functions) -- @@ -407,6 +411,20 @@ async def _run_pre_llm_steps( if not no_relevant_context and not filtered: no_relevant_context = True + # Step 6.5: Parent chunk expansion (FEAT-017) + if self._parent_expansion and filtered: + t0 = time.monotonic() + filtered, expansion_meta = await self._expand_parents( + query.namespace, filtered + ) + steps.append( + StepTrace( + name="parent_expansion", + duration_ms=_elapsed_ms(t0), + metadata=expansion_meta, + ) + ) + return ( steps, filtered, @@ -416,6 +434,59 @@ async def _run_pre_llm_steps( history, ) + async def _expand_parents( + self, + namespace: str, + results: list[SearchResult], + ) -> tuple[list[SearchResult], dict[str, object]]: + """Replace child chunk text with the parent chunk text (FEAT-017). + + Children of the same parent collapse into a single result (the + highest-scored one, since results arrive score-ordered): the parent + text already contains all its children. Runs before token budgeting + (ARCH-055) so the budget sees the expanded text. Results keep the + child's chunk_id and score for trace comparability. + """ + parent_ids: list[str] = [] + for r in results: + if r.parent_id and r.parent_id not in parent_ids: + parent_ids.append(r.parent_id) + if not parent_ids: + return results, {"children_expanded": 0, "parents_fetched": 0} + + try: + parents = await self._vector_store.retrieve( + namespace=namespace, chunk_ids=parent_ids + ) + except Exception as exc: + log.warning("parent_expansion_failed", error=str(exc)) + return results, {"skipped": True, "error": str(exc)} + + parent_text = {p.chunk_id: p.text_snippet for p in parents} + expanded: list[SearchResult] = [] + seen_parents: set[str] = set() + children_expanded = 0 + siblings_merged = 0 + for r in results: + if r.parent_id and r.parent_id in parent_text: + if r.parent_id in seen_parents: + siblings_merged += 1 + continue + seen_parents.add(r.parent_id) + children_expanded += 1 + expanded.append( + dataclasses.replace(r, text_snippet=parent_text[r.parent_id]) + ) + else: + expanded.append(r) + + return expanded, { + "children_expanded": children_expanded, + "siblings_merged": siblings_merged, + "parents_fetched": len(parents), + "after_expansion": len(expanded), + } + def _build_prompt( self, query: QueryRequest, diff --git a/vektra-index/src/vektra_index/adapters.py b/vektra-index/src/vektra_index/adapters.py index 4b4d184d..efb67280 100644 --- a/vektra-index/src/vektra_index/adapters.py +++ b/vektra-index/src/vektra_index/adapters.py @@ -120,6 +120,17 @@ async def search( raw_filters=raw_filters, ) + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + factory = self._get_session_factory() + pgvector = self._get_pgvector() + + async with factory() as session: + return await pgvector.retrieve(session, namespace, chunk_ids) + async def delete(self, namespace: str, ids: list[str]) -> int: """Delete all chunks for each document_id in ids. diff --git a/vektra-index/src/vektra_index/providers/pgvector.py b/vektra-index/src/vektra_index/providers/pgvector.py index 62f4cb03..7834fa83 100644 --- a/vektra-index/src/vektra_index/providers/pgvector.py +++ b/vektra-index/src/vektra_index/providers/pgvector.py @@ -73,7 +73,18 @@ async def store( inserted_ids: list[str] = [] for position, chunk in enumerate(chunks): - chunk_id = uuid4() + # Honor caller-provided deterministic ids (uuid5 from ingest); + # parent linkage relies on them (FEAT-017). + try: + chunk_id = UUID(chunk.chunk_id) if chunk.chunk_id else uuid4() + except ValueError: + chunk_id = uuid4() + parent_uuid: UUID | None = None + if chunk.parent_id: + try: + parent_uuid = UUID(chunk.parent_id) + except ValueError: + parent_uuid = None sparse_data = None if chunk.sparse is not None: sparse_data = { @@ -90,6 +101,7 @@ async def store( chunk_metadata=chunk.metadata, position=chunk.metadata.get("position", position), index_version=self._active_index_version, + parent_id=parent_uuid, ) session.add(orm_obj) inserted_ids.append(str(chunk_id)) @@ -151,6 +163,7 @@ async def _search_dense( DocumentChunkOrm.document_id, DocumentChunkOrm.content, DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, score_expr, ) .where( @@ -161,6 +174,7 @@ async def _search_dense( .limit(top_k) ) + stmt = self._exclude_parent_chunks(stmt) stmt = self._apply_filters(stmt, filters) stmt = self._join_source_documents(stmt) @@ -228,6 +242,7 @@ async def _search_sparse( DocumentChunkOrm.document_id, DocumentChunkOrm.content, DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, score_col, ) .where( @@ -239,6 +254,7 @@ async def _search_sparse( .limit(top_k) ) + stmt = self._exclude_parent_chunks(stmt) stmt = self._apply_filters(stmt, filters) stmt = self._join_source_documents(stmt) @@ -314,10 +330,27 @@ async def _search_hybrid( document_id=r.document_id, document_version=r.document_version, metadata=r.metadata, + parent_id=r.parent_id, ) for rrf_score, r in rrf_scored[:top_k] ] + @staticmethod + def _exclude_parent_chunks(stmt: Any) -> Any: + """Exclude parent-level chunks from search results (FEAT-017). + + Parent chunks are context material fetched by id during expansion, + not retrieval targets. Chunks without a chunk_level key (fixed + chunking) are unaffected: NULL IS DISTINCT FROM 'parent'. + """ + from vektra_index.models import DocumentChunkOrm + + return stmt.where( + DocumentChunkOrm.chunk_metadata["chunk_level"] + .as_string() + .is_distinct_from("parent") + ) + @staticmethod def _apply_filters(stmt: Any, filters: SearchFilters | None) -> Any: """Apply JSONB metadata filters in the same SQL query (REQ-063).""" @@ -361,10 +394,48 @@ def _rows_to_results(rows: Sequence[Any]) -> list[SearchResult]: document_id=row.document_id, document_version=row.document_version, metadata=row.chunk_metadata or {}, + parent_id=str(row.parent_id) if row.parent_id else None, ) for row in rows ] + async def retrieve( + self, + session: AsyncSession, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + """Fetch chunks by id (no vector search). Used for parent chunk + expansion (FEAT-017); score is 0.0 by convention. + """ + from vektra_index.models import DocumentChunkOrm + + valid_ids: list[UUID] = [] + for cid in chunk_ids: + try: + valid_ids.append(UUID(cid)) + except ValueError: + logger.warning("retrieve_invalid_chunk_id: %s", cid) + if not valid_ids: + return [] + + stmt = select( + DocumentChunkOrm.id, + DocumentChunkOrm.document_id, + DocumentChunkOrm.content, + DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, + literal_column("0.0").label("score"), + ).where( + DocumentChunkOrm.id.in_(valid_ids), + DocumentChunkOrm.namespace_id == namespace, + DocumentChunkOrm.index_version == self._active_index_version, + ) + stmt = self._join_source_documents(stmt) + + result = await session.execute(stmt) + return self._rows_to_results(result.all()) + async def delete( self, session: AsyncSession, diff --git a/vektra-index/src/vektra_index/providers/qdrant.py b/vektra-index/src/vektra_index/providers/qdrant.py index bb27d8ae..9904dbef 100644 --- a/vektra-index/src/vektra_index/providers/qdrant.py +++ b/vektra-index/src/vektra_index/providers/qdrant.py @@ -184,6 +184,7 @@ async def store( "text": chunk.text, "metadata": chunk.metadata, "document_id": chunk.metadata.get("document_id", ""), + "parent_id": chunk.parent_id, }, ) ) @@ -333,6 +334,30 @@ async def _search_hybrid( ) return self._points_to_results(results.points) + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + """Fetch points by id (no vector search). Used for parent chunk + expansion (FEAT-017); score is 0.0 by convention. + + Points whose payload namespace does not match are dropped (namespace + isolation): Qdrant retrieve() takes no filter. + """ + if not chunk_ids: + return [] + + records = await self._client.retrieve( + collection_name=self._collection_name, + ids=chunk_ids, + with_payload=True, + ) + matching = [ + r for r in records if (r.payload or {}).get("namespace_id") == namespace + ] + return self._points_to_results(matching) + async def delete(self, namespace: str, ids: list[str]) -> int: """Delete points by document_id payload filter. @@ -428,11 +453,24 @@ def _build_filter( ) ) - return models.Filter(must=must_conditions) + # Parent chunks are context material, not retrieval targets (FEAT-017) + return models.Filter( + must=must_conditions, + must_not=[ + models.FieldCondition( + key="metadata.chunk_level", + match=models.MatchValue(value="parent"), + ) + ], + ) @staticmethod def _points_to_results(points: list[Any]) -> list[SearchResult]: - """Convert Qdrant ScoredPoint list to SearchResult list.""" + """Convert Qdrant ScoredPoint/Record list to SearchResult list. + + Record objects (from retrieve()) have no score attribute: score + defaults to 0.0. + """ results: list[SearchResult] = [] for point in points: payload = point.payload or {} @@ -442,16 +480,18 @@ def _points_to_results(points: list[Any]) -> list[SearchResult]: except (ValueError, AttributeError): doc_id = UUID(int=0) + score = getattr(point, "score", None) results.append( SearchResult( chunk_id=str(point.id), - score=float(point.score) if point.score is not None else 0.0, + score=float(score) if score is not None else 0.0, text_snippet=payload.get("text", ""), document_id=doc_id, document_version=payload.get("metadata", {}).get( "document_version", 1 ), metadata=payload.get("metadata", {}), + parent_id=payload.get("parent_id"), ) ) return results diff --git a/vektra-ingest/src/vektra_ingest/chunking.py b/vektra-ingest/src/vektra_ingest/chunking.py index 194c67b2..2d45cd0c 100644 --- a/vektra-ingest/src/vektra_ingest/chunking.py +++ b/vektra-ingest/src/vektra_ingest/chunking.py @@ -265,6 +265,9 @@ async def _chunk_impl( parent_start = parent_end continue + # The parent carries its own hierarchy id so the ingest + # pipeline can remap these chunker-local ids to stored + # chunk ids (FEAT-017). yield DocumentChunk( text=parent_text, element_type=ElementType.TEXT, @@ -274,7 +277,7 @@ async def _chunk_impl( "token_count": len(parent_tokens), "chunk_level": "parent", }, - parent_id=None, + parent_id=parent_id, ) chunk_index += 1 diff --git a/vektra-ingest/src/vektra_ingest/pipeline.py b/vektra-ingest/src/vektra_ingest/pipeline.py index 4a3663ee..4c8be799 100644 --- a/vektra-ingest/src/vektra_ingest/pipeline.py +++ b/vektra-ingest/src/vektra_ingest/pipeline.py @@ -395,6 +395,15 @@ async def run_ingest( ), ) + # Map chunker-local hierarchy ids to stored chunk ids (FEAT-017). + # Parent chunks carry their own transient id in chunk.parent_id; + # children reference it. Stored ids are deterministic: uuid5(doc_id, position). + parent_stored_ids = { + chunk.parent_id: str(uuid5(doc_id, str(i))) + for i, chunk in enumerate(all_chunks) + if chunk.parent_id and chunk.metadata.get("chunk_level") == "parent" + } + # Build ChunkEmbedding objects with document_id in metadata _extra = extra_metadata or {} chunk_embeddings = [ @@ -411,6 +420,11 @@ async def run_ingest( "element_type": chunk.element_type.value, "position": i, }, + parent_id=( + parent_stored_ids.get(chunk.parent_id) + if chunk.parent_id and chunk.metadata.get("chunk_level") == "child" + else None + ), ) for i, (chunk, embedding, sparse) in enumerate( zip(all_chunks, embeddings, sparse_vectors) diff --git a/vektra-ingest/tests/test_dual_chunking.py b/vektra-ingest/tests/test_dual_chunking.py index 7222dc92..d92d343a 100644 --- a/vektra-ingest/tests/test_dual_chunking.py +++ b/vektra-ingest/tests/test_dual_chunking.py @@ -74,20 +74,17 @@ async def test_text_elements_split_with_overlap(): chunks = await _collect(chunker, elements) # Should have at least one parent and at least one child - parents = [ - c for c in chunks if c.parent_id is None and c.element_type == ElementType.TEXT - ] - children = [c for c in chunks if c.parent_id is not None] + parents = [c for c in chunks if c.metadata.get("chunk_level") == "parent"] + children = [c for c in chunks if c.metadata.get("chunk_level") == "child"] assert len(parents) >= 1 assert len(children) >= 1 + # Parents carry their own hierarchy id; children reference it (FEAT-017) + parent_ids = {p.parent_id for p in parents} + assert None not in parent_ids for child in children: - assert child.parent_id is not None - assert child.metadata.get("chunk_level") == "child" - - for parent in parents: - assert parent.metadata.get("chunk_level") == "parent" + assert child.parent_id in parent_ids @pytest.mark.asyncio diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 5b602fae..987b5903 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -197,6 +197,11 @@ class QueryPipelineConfig(BaseSettings): alias="VEKTRA_CHUNK_DEDUP_ENABLED", description="Enable overlap deduplication for adjacent chunks from the same document.", ) + parent_expansion_enabled: bool = Field( + False, + alias="VEKTRA_PARENT_EXPANSION_ENABLED", + description="Replace retrieved child chunks with their parent chunk text before prompt construction (FEAT-017). Requires dual chunking at ingest time.", + ) response_token_reserve: int = Field( 2048, ge=1, @@ -493,6 +498,9 @@ class VektraSettings(BaseSettings): query_pipeline: str = Field("advanced", alias="VEKTRA_QUERY_PIPELINE") min_relevance_score: float = Field(0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE") chunk_dedup_enabled: bool = Field(True, alias="VEKTRA_CHUNK_DEDUP_ENABLED") + parent_expansion_enabled: bool = Field( + False, alias="VEKTRA_PARENT_EXPANSION_ENABLED" + ) response_token_reserve: int = Field(2048, alias="VEKTRA_RESPONSE_TOKEN_RESERVE") context_chunk_ratio: float = Field(0.6, alias="VEKTRA_CONTEXT_CHUNK_RATIO") prompt_templates_dir: str | None = Field(None, alias="VEKTRA_PROMPT_TEMPLATES_DIR") diff --git a/vektra-shared/src/vektra_shared/protocols.py b/vektra-shared/src/vektra_shared/protocols.py index 7c633f11..1415d9bc 100644 --- a/vektra-shared/src/vektra_shared/protocols.py +++ b/vektra-shared/src/vektra_shared/protocols.py @@ -117,6 +117,12 @@ async def search( raw_filters: dict[str, Any] | None = None, ) -> list[SearchResult]: ... + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: ... + async def delete(self, namespace: str, ids: list[str]) -> int: ... async def health_check(self) -> HealthStatus: ... diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index d1d90342..38729007 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -121,6 +121,7 @@ class ChunkEmbedding: dense: list[float] sparse: SparseVector | None = None # Phase 2: hybrid search metadata: dict[str, Any] = field(default_factory=dict) + parent_id: str | None = None # stored chunk_id of the parent chunk (FEAT-017) # --------------------------------------------------------------------------- @@ -157,6 +158,7 @@ class SearchResult: document_version: int = 1 # from SourceDocument.version (REQ-056) metadata: dict[str, Any] = field(default_factory=dict) original_score: float | None = None # pre-reranker score (BUG-015) + parent_id: str | None = None # stored chunk_id of the parent chunk (FEAT-017) @dataclass From 48c0594663980f82f7c495f39a2891d340c24c99 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 21:21:23 +0000 Subject: [PATCH 15/31] test(rag): cover parent linkage, search exclusion, and expansion (FEAT-017) - ingest: dual ingest maps children to the parent's deterministic stored id (uuid5(doc_id, position)); parents and tables carry no parent_id - qdrant: parent_id in payload, must_not filter on chunk_level=parent, retrieve() namespace isolation and Record (no score) handling - pgvector: caller-provided uuid ids honored, parent_id column filled, dense search excludes parents, retrieve() maps rows and skips bad ids - advanced pipeline: expansion replaces child text with parent text, merges siblings, degrades gracefully on retrieve failure, keeps children when the parent is missing, and feeds the token budget with the expanded text; default-off leaves the pipeline untouched Co-Authored-By: Claude Fable 5 --- vektra-core/tests/test_advanced_pipeline.py | 189 ++++++++++++++++++++ vektra-index/tests/test_pgvector_unit.py | 141 +++++++++++++++ vektra-index/tests/test_qdrant_provider.py | 126 +++++++++++++ vektra-ingest/tests/test_pipeline.py | 121 +++++++++++++ 4 files changed, 577 insertions(+) diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index bdd1c488..2956340c 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -770,3 +770,192 @@ async def test_eval_mode_off_excludes_prompt_messages(): build_step = next(s for s in trace.steps if s.name == "build_prompt") assert "messages" not in build_step.metadata + + +# --------------------------------------------------------------------------- +# Parent chunk expansion (FEAT-017) +# --------------------------------------------------------------------------- + + +def _make_child_result( + score: float, text: str, parent_id: str, doc_id: UUID | None = None +) -> SearchResult: + return SearchResult( + chunk_id=str(uuid4()), + score=score, + text_snippet=text, + document_id=doc_id or uuid4(), + document_version=1, + parent_id=parent_id, + ) + + +def _make_parent_chunk(chunk_id: str, text: str, doc_id: UUID) -> SearchResult: + return SearchResult( + chunk_id=chunk_id, + score=0.0, + text_snippet=text, + document_id=doc_id, + document_version=1, + ) + + +async def test_parent_expansion_replaces_child_text(): + """Expanded children carry the parent text into prompt and sources.""" + doc_id = uuid4() + parent_text = "PARENT SECTION: full surrounding context for the child." + child = _make_child_result(0.9, "tiny child snippet", "parent-1", doc_id) + orphan = _make_search_result(0.8, "standalone chunk without parent") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child, orphan]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute( + QueryRequest(question="test", namespace="default") + ) + + vector_store.retrieve.assert_awaited_once_with( + namespace="default", chunk_ids=["parent-1"] + ) + # Expanded result keeps the child's chunk_id and score + by_id = {s.chunk_id: s for s in response.sources} + assert by_id[child.chunk_id].snippet == parent_text + assert by_id[child.chunk_id].score == child.score + assert by_id[orphan.chunk_id].snippet == orphan.text_snippet + + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 1 + assert step.metadata["siblings_merged"] == 0 + assert step.metadata["parents_fetched"] == 1 + + +async def test_parent_expansion_merges_siblings_of_same_parent(): + """Children of the same parent collapse into the highest-scored one.""" + doc_id = uuid4() + parent_text = "PARENT: contains both sibling fragments." + sibling_hi = _make_child_result(0.9, "fragment one", "parent-1", doc_id) + sibling_lo = _make_child_result(0.7, "fragment two", "parent-1", doc_id) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[sibling_hi, sibling_lo]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert len(response.sources) == 1 + assert response.sources[0].chunk_id == sibling_hi.chunk_id + assert response.sources[0].snippet == parent_text + + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 1 + assert step.metadata["siblings_merged"] == 1 + + +async def test_parent_expansion_disabled_by_default(): + """Without the flag no retrieve call is made and no step is traced.""" + child = _make_child_result(0.9, "child snippet", "parent-1") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + + pipeline = _make_pipeline(vector_store=vector_store) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + vector_store.retrieve.assert_not_awaited() + assert all(s.name != "parent_expansion" for s in trace.steps) + assert response.sources[0].snippet == "child snippet" + + +async def test_parent_expansion_retrieve_failure_keeps_children(): + """A vector store failure during expansion degrades gracefully.""" + child = _make_child_result(0.9, "child snippet", "parent-1") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock(side_effect=RuntimeError("qdrant down")) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert response.answer is not None + assert response.sources[0].snippet == "child snippet" + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata.get("skipped") is True + + +async def test_parent_expansion_missing_parent_keeps_child(): + """Children whose parent is not returned by retrieve() stay unexpanded.""" + child = _make_child_result(0.9, "child snippet", "parent-gone") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock(return_value=[]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert response.sources[0].snippet == "child snippet" + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 0 + assert step.metadata["parents_fetched"] == 0 + + +async def test_parent_expansion_feeds_token_budget_with_parent_text(): + """Token budgeting (ARCH-055) operates on the expanded parent text.""" + doc_id = uuid4() + parent_text = "PARENT TEXT " * 50 + child = _make_child_result(0.9, "tiny", "parent-1", doc_id) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + llm = MagicMock() + llm.complete = AsyncMock( + return_value=CompletionResponse( + content="The answer.", + model="ollama/llama3", + prompt_tokens=10, + completion_tokens=20, + total_tokens=30, + ) + ) + token_counts: list[str] = [] + + def _count(text: str, model: str | None = None) -> int: + token_counts.append(text) + return len(text.split()) + + llm.count_tokens = MagicMock(side_effect=_count) + + pipeline = _make_pipeline( + vector_store=vector_store, + llm=llm, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + await pipeline.execute(QueryRequest(question="test")) + + # The budget counted the parent text, not the child snippet + assert any(parent_text == t for t in token_counts) + assert all(t != "tiny" for t in token_counts) diff --git a/vektra-index/tests/test_pgvector_unit.py b/vektra-index/tests/test_pgvector_unit.py index 19233c1d..e8e8e06f 100644 --- a/vektra-index/tests/test_pgvector_unit.py +++ b/vektra-index/tests/test_pgvector_unit.py @@ -163,3 +163,144 @@ async def test_custom_index_version_used(self): provider = PgvectorProvider(active_index_version=2) assert provider._active_index_version == 2 + + +class TestPgvectorParentChunks: + """Deterministic ids, parent linkage, search exclusion, retrieve (FEAT-017).""" + + def _make_session(self): + session = AsyncMock() + session.flush = AsyncMock() + session.execute = AsyncMock() + session.add = MagicMock() + return session + + @pytest.mark.asyncio + async def test_store_honors_deterministic_ids_and_parent_linkage(self): + """store() keeps caller-provided uuid ids and fills the parent_id column.""" + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + parent_uuid = uuid4() + child_uuid = uuid4() + chunks = [ + ChunkEmbedding( + chunk_id=str(parent_uuid), + text="parent", + dense=[0.1] * 384, + metadata={"chunk_level": "parent"}, + ), + ChunkEmbedding( + chunk_id=str(child_uuid), + text="child", + dense=[0.1] * 384, + metadata={"chunk_level": "child"}, + parent_id=str(parent_uuid), + ), + ] + + with patch("vektra_index.models.DocumentChunkOrm") as MockOrm: + MockOrm.return_value = MagicMock() + result = await provider.store(session, "default", uuid4(), chunks) + + assert result == [str(parent_uuid), str(child_uuid)] + orm_kwargs = [c.kwargs for c in MockOrm.call_args_list] + assert orm_kwargs[0]["id"] == parent_uuid + assert orm_kwargs[0]["parent_id"] is None + assert orm_kwargs[1]["id"] == child_uuid + assert orm_kwargs[1]["parent_id"] == parent_uuid + + @pytest.mark.asyncio + async def test_store_falls_back_to_random_id_on_non_uuid(self): + """Non-UUID caller ids (external store-chunks callers) get random uuids.""" + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + chunks = [ + ChunkEmbedding( + chunk_id="not-a-uuid", + text="x", + dense=[0.1] * 384, + metadata={}, + parent_id="also-not-a-uuid", + ), + ] + + with patch("vektra_index.models.DocumentChunkOrm") as MockOrm: + MockOrm.return_value = MagicMock() + result = await provider.store(session, "default", uuid4(), chunks) + + UUID(result[0]) # random but valid + assert MockOrm.call_args.kwargs["parent_id"] is None + + @pytest.mark.asyncio + async def test_dense_search_excludes_parent_chunks(self): + """The dense search SQL filters out chunk_level=parent rows.""" + from sqlalchemy.dialects import postgresql + + from vektra_index.providers.pgvector import PgvectorProvider + from vektra_shared.types import QueryEmbedding + + session = self._make_session() + empty_result = MagicMock() + empty_result.all.return_value = [] + session.execute = AsyncMock(return_value=empty_result) + + provider = PgvectorProvider() + await provider.search( + session, "default", QueryEmbedding(dense=[0.1] * 384), top_k=5 + ) + + stmt = session.execute.call_args.args[0] + compiled = stmt.compile(dialect=postgresql.dialect()) + assert "IS DISTINCT FROM" in str(compiled) + # The JSONB accessor key and the excluded value are bind parameters + assert "chunk_level" in compiled.params.values() + assert "parent" in compiled.params.values() + + @pytest.mark.asyncio + async def test_retrieve_maps_rows_and_skips_invalid_ids(self): + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + parent_uuid = uuid4() + doc_id = uuid4() + row = FakeRow( + id=parent_uuid, + document_id=doc_id, + content="parent text", + chunk_metadata={"chunk_level": "parent"}, + parent_id=None, + score=0.0, + document_version=1, + ) + rows_result = MagicMock() + rows_result.all.return_value = [row] + session.execute = AsyncMock(return_value=rows_result) + + provider = PgvectorProvider() + results = await provider.retrieve( + session, "default", [str(parent_uuid), "not-a-uuid"] + ) + + assert len(results) == 1 + assert results[0].chunk_id == str(parent_uuid) + assert results[0].score == 0.0 + assert results[0].text_snippet == "parent text" + assert results[0].parent_id is None + + @pytest.mark.asyncio + async def test_retrieve_all_invalid_ids_returns_empty(self): + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + results = await provider.retrieve(session, "default", ["nope", "still-nope"]) + + assert results == [] + session.execute.assert_not_called() diff --git a/vektra-index/tests/test_qdrant_provider.py b/vektra-index/tests/test_qdrant_provider.py index 686f6ffd..0b5ffe16 100644 --- a/vektra-index/tests/test_qdrant_provider.py +++ b/vektra-index/tests/test_qdrant_provider.py @@ -244,3 +244,129 @@ async def test_unhealthy(self): status = await provider.health_check() assert status.status == "unhealthy" assert "connection refused" in status.message + + +class TestQdrantParentChunks: + """Parent chunk linkage, search exclusion, and retrieve-by-id (FEAT-017).""" + + @pytest.mark.asyncio + async def test_store_includes_parent_id_in_payload(self): + provider, _client = _make_provider() + doc_id = str(uuid4()) + + _mock_qdrant_models.PointStruct.reset_mock() + chunks = [ + ChunkEmbedding( + chunk_id="child-1", + text="child text", + dense=[0.1] * 384, + metadata={"document_id": doc_id, "chunk_level": "child"}, + parent_id="parent-1", + ), + ChunkEmbedding( + chunk_id="parent-1", + text="parent text", + dense=[0.1] * 384, + metadata={"document_id": doc_id, "chunk_level": "parent"}, + ), + ] + + await provider.store("default", chunks) + + payloads = [ + c.kwargs["payload"] for c in _mock_qdrant_models.PointStruct.call_args_list + ] + assert payloads[0]["parent_id"] == "parent-1" + assert payloads[1]["parent_id"] is None + + @pytest.mark.asyncio + async def test_build_filter_excludes_parent_chunks(self): + provider, _client = _make_provider() + + _mock_qdrant_models.Filter.reset_mock() + _mock_qdrant_models.FieldCondition.reset_mock() + provider._build_filter("default") + + filter_kwargs = _mock_qdrant_models.Filter.call_args.kwargs + assert "must_not" in filter_kwargs + assert len(filter_kwargs["must_not"]) == 1 + condition_keys = [ + c.kwargs.get("key") + for c in _mock_qdrant_models.FieldCondition.call_args_list + ] + assert "metadata.chunk_level" in condition_keys + + @pytest.mark.asyncio + async def test_search_results_carry_parent_id(self): + provider, client = _make_provider() + doc_id = str(uuid4()) + + query_result = MagicMock() + query_result.points = [ + _make_scored_point( + "child-1", + 0.9, + { + "text": "child text", + "document_id": doc_id, + "metadata": {"chunk_level": "child"}, + "namespace_id": "default", + "parent_id": "parent-1", + }, + ), + ] + client.query_points = AsyncMock(return_value=query_result) + + results = await provider.search( + "default", QueryEmbedding(dense=[0.1] * 384), 5, SearchMode.DENSE + ) + + assert results[0].parent_id == "parent-1" + + @pytest.mark.asyncio + async def test_retrieve_filters_by_namespace(self): + from types import SimpleNamespace + + provider, client = _make_provider() + doc_id = str(uuid4()) + + # Qdrant retrieve() returns Record objects with no score attribute + records = [ + SimpleNamespace( + id="parent-1", + payload={ + "text": "parent text", + "document_id": doc_id, + "metadata": {"chunk_level": "parent"}, + "namespace_id": "default", + "parent_id": None, + }, + ), + SimpleNamespace( + id="alien-1", + payload={ + "text": "other tenant", + "document_id": doc_id, + "metadata": {}, + "namespace_id": "other", + "parent_id": None, + }, + ), + ] + client.retrieve = AsyncMock(return_value=records) + + results = await provider.retrieve("default", ["parent-1", "alien-1"]) + + client.retrieve.assert_awaited_once() + assert [r.chunk_id for r in results] == ["parent-1"] + assert results[0].score == 0.0 + assert results[0].text_snippet == "parent text" + + @pytest.mark.asyncio + async def test_retrieve_empty_ids_short_circuits(self): + provider, client = _make_provider() + + results = await provider.retrieve("default", []) + + assert results == [] + client.retrieve.assert_not_called() diff --git a/vektra-ingest/tests/test_pipeline.py b/vektra-ingest/tests/test_pipeline.py index 76236ac7..2968be15 100644 --- a/vektra-ingest/tests/test_pipeline.py +++ b/vektra-ingest/tests/test_pipeline.py @@ -711,3 +711,124 @@ async def _gen(): session=session, registry=registry, ) + + +# --------------------------------------------------------------------------- +# Parent chunk linkage at store time (FEAT-017) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_dual_ingest_links_children_to_stored_parent_ids(monkeypatch): + """Children reference the parent's deterministic stored id, not the + chunker-local transient id; parents and tables carry no parent_id.""" + from uuid import uuid5 + + from vektra_ingest.pipeline import run_ingest + + monkeypatch.setenv("VEKTRA_CHUNKING_STRATEGY", "dual") + monkeypatch.setenv("VEKTRA_PARENT_CHILD_LEVELS", "1") + + doc_id = uuid4() + + session = AsyncMock() + + async def _execute(stmt, *args, **kwargs): + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = None + return mock_result + + added_docs: list = [] + + def _session_add(obj): + added_docs.append(obj) + + async def _flush_side_effect(): + for obj in added_docs: + obj.id = doc_id + + session.execute = _execute + session.add = _session_add + session.flush = AsyncMock(side_effect=_flush_side_effect) + session.commit = AsyncMock() + + registry, mock_embedding, mock_vs = _make_registry() + mock_embedding.embed_documents = AsyncMock(return_value=[[0.1] * 384] * 4) + mock_vs.store = AsyncMock(return_value=["i0", "i1", "i2", "i3"]) + + async def _fake_extract(req): + async def _gen(): + yield DocumentChunk(text="whatever", element_type=ElementType.TEXT) + + return _gen() + + # Controlled dual output: parent + two children + standalone table + async def _fake_chunk(elements): + async def _gen(): + yield DocumentChunk( + text="parent text", + element_type=ElementType.TEXT, + metadata={"chunk_level": "parent"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="child a", + element_type=ElementType.TEXT, + metadata={"chunk_level": "child"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="child b", + element_type=ElementType.TEXT, + metadata={"chunk_level": "child"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="standalone
", + element_type=ElementType.TABLE, + metadata={}, + parent_id=None, + ) + + return _gen() + + mock_extractor = MagicMock() + mock_extractor.extract = _fake_extract + mock_extractor.supported_types.return_value = {"application/pdf"} + + with patch( + "vektra_ingest.pipeline.detect_content_type", return_value="application/pdf" + ): + with patch( + "vektra_ingest.pipeline._get_extractor", return_value=mock_extractor + ): + with patch( + "vektra_ingest.pipeline.DualStrategyChunking" + ) as mock_chunker_class: + mock_chunker = MagicMock() + mock_chunker.chunk = _fake_chunk + mock_chunker_class.return_value = mock_chunker + + result = await run_ingest( + file_content=b"%PDF-1.4 fake pdf", + filename="dual.pdf", + namespace="default", + session=session, + registry=registry, + ) + + assert result.status == "new" + stored_chunks = mock_vs.store.call_args.args[1] + assert len(stored_chunks) == 4 + + parent, child_a, child_b, table = stored_chunks + expected_parent_stored_id = str(uuid5(doc_id, "0")) + + assert parent.chunk_id == expected_parent_stored_id + assert parent.parent_id is None + assert child_a.parent_id == expected_parent_stored_id + assert child_b.parent_id == expected_parent_stored_id + assert table.parent_id is None + # Deterministic ids for every chunk: uuid5(doc_id, position) + for i, chunk in enumerate(stored_chunks): + assert chunk.chunk_id == str(uuid5(doc_id, str(i))) From ddbddcb5ca6ca069674014e0e237e6bd4163176c Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 21:48:09 +0000 Subject: [PATCH 16/31] docs(s2s): record FEAT-017 A/B measurement; file TECH-007; extend DEBT-025 fixture - plan: section 3 complete; Notes with measure A (dual, no expansion: retrieval -6.5pp hit from parent-boundary chunking, e2e unchanged) and measure B (expansion on: grounding-neutral, zero flips, sources 1.4->1.0 via sibling merge, expansion verified in traces) - backlog: FEAT-017 completed with premise corrections and honest numbers; TECH-007 filed with the multi-chunk collapse root cause (rerank scores for multi-part questions all below min_relevance_score 0.15; MC-01 max rerank 0.088 vs raw RRF 0.61 - the funnel wipes candidates before expansion can run) - changelog: FEAT-017 Added entry + Changed entries (parent exclusion in search, pgvector deterministic ids) - tests: DEBT-025 hermetic fixture replicated in vektra-core and vektra-ingest (ambient VEKTRA_CHUNKING_STRATEGY=dual from the dev .env broke 12 tests there; vektra-shared alone was covered) Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 35 ++++++++++--- .s2s/plans/20260712-sprint3-rag-quality.md | 57 +++++++++++++++++----- CHANGELOG.md | 11 ++++- vektra-core/tests/conftest.py | 26 ++++++++++ vektra-ingest/tests/conftest.py | 26 ++++++++++ 5 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 vektra-core/tests/conftest.py create mode 100644 vektra-ingest/tests/conftest.py diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 1aab8686..d048fd96 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -283,19 +283,23 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. ### FEAT-017: Parent chunk expansion in query pipeline -**Status**: planned | **Priority**: medium | **Created**: 2026-03-23 +**Status**: completed (2026-07-12) | **Priority**: medium | **Created**: 2026-03-23 **Analysis**: `vektra-internal/stack/20260323-rag-prompt-chunk-confusion-analysis.md` **Context**: when a child chunk is retrieved via search, the pipeline should optionally expand it to the parent chunk for broader context. The infrastructure is already in place: `DualStrategyChunking` creates parent-child hierarchy (parent every 3000 tokens, children at 500 tokens with overlap), `DocumentChunkOrm` has `parent_id` column, and both are stored in the database. Missing: (1) filter parent chunks from default search results (search currently returns both), (2) parent expansion logic in AdvancedQueryPipeline when a child matches. +**Premise corrections found during implementation**: the hierarchy was NOT actually persisted anywhere (`run_ingest` dropped `chunk.parent_id`, `ChunkEmbedding` had no field, Qdrant payload had none, `DocumentChunkOrm.parent_id` was always NULL); parent size is `chunk_size*3` (1500 tokens with Combo D 500, not 3000); pgvector `store()` ignored caller chunk ids (generated uuid4), so deterministic ids had to be plumbed there too. + **Traceability**: ARCH-037 (ChunkingStrategy), ARCH-055 (token budget), core-pipeline-v2 **Acceptance criteria**: -- [ ] Search excludes parent chunks by default (WHERE parent_id IS NOT NULL for children only) -- [ ] AdvancedQueryPipeline fetches parent chunk when child matches and includes it in context -- [ ] Parent expansion is configurable (on/off, via env var) -- [ ] Token budget accounts for expanded parent chunk size -- [ ] Tested: truncated-context answers improve with parent expansion enabled +- [x] Search excludes parent chunks by default (`chunk_level=parent` filtered: Qdrant `must_not`, pgvector `IS DISTINCT FROM`) +- [x] AdvancedQueryPipeline fetches parent chunk when child matches and includes it in context (step 6.5, new `VectorStoreProvider.retrieve()`) +- [x] Parent expansion is configurable (`VEKTRA_PARENT_EXPANSION_ENABLED`, default off) +- [x] Token budget accounts for expanded parent chunk size (expansion runs before ARCH-055 allocation) +- [x] Tested: 17 unit tests (linkage, exclusion, expansion, budget); measured on `eval-full` — expansion is grounding-neutral there (35/55 with and without, zero per-question flips, avg sources 1.4 → 1.0 via sibling merge). The multi-chunk collapse it targeted turned out to be upstream: the rerank+threshold funnel wipes all candidates before expansion (TECH-007). The corpus also understates expansion benefit (short self-contained articles — TECH-005 collection 1 is the real test bench). + +**Resolution (2026-07-12, Sprint 3)**: shipped default-off on `feat/feat-017-parent-chunk-expansion`. Measured A/B on `eval-full` reingested dual (105 points = 22 parents + 83 children): dual chunking alone costs ~6.5pp retrieval hit vs fixed (children stop rolling overlap across 1500-token parent boundaries; IT-F-13, IT-R-02, EN-F-03 flip to miss). Full numbers in plan `20260712-sprint3-rag-quality` Notes. --- @@ -357,6 +361,25 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. --- +### TECH-007: Multi-part questions wiped by rerank+threshold funnel (multi-chunk collapse root cause) + +**Status**: planned | **Priority**: high | **Created**: 2026-07-12 +**Origin**: FEAT-017 measurement (plan `20260712-sprint3-rag-quality`) - expansion turned out to be downstream of the real failure. + +**Context**: on `eval-full`, 9/10 multi-chunk questions end with `retrieval_filter before=5 after=0` → `no_relevant_context` → refusal, despite 90% raw retrieval hit for the category. Cause: bge-reranker-v2-m3 scores each partial-answer chunk of a comparative/multi-part question low (each chunk answers only one part), and `VEKTRA_MIN_RELEVANCE_SCORE=0.15` — calibrated in the tuning sprint on single-fact questions (DEBT-010) — wipes the entire candidate set. Evidence (MC-01, eval mode traces): max reranker score 0.088 on a candidate whose raw RRF score was 0.61. Parent expansion (FEAT-017) never runs because zero results survive the filter. + +**Candidate directions** (evaluate, do not assume): (a) floor semantics - keep top-N post-rerank chunks regardless of threshold when the raw retrieval score was strong (e.g. min(top_k, after_rerank) >= 2); (b) per-category or per-score-source thresholds (reranker scores are not calibrated on the same scale as RRF); (c) query decomposition for multi-part questions (rewrite step already exists, ARCH-061); (d) rescore against the parent text instead of the child (combines with FEAT-017). + +**Traceability**: ARCH-056 (retrieval quality controls), ADR-0021, DEBT-010, FEAT-017, TECH-005 + +**Acceptance criteria**: +- [ ] Reproduce with the eval harness and document the score distributions per category +- [ ] Chosen mitigation implemented behind config, default preserving current single-fact behavior +- [ ] `eval-full` multi-chunk grounded moves from 0-1/10 without regressing factual (19/21) or adversarial refusals (no answered-without-context) +- [ ] Decision and numbers recorded in the sprint plan and vektra-internal + +--- + ### FEAT-024: Remote embedding and reranker providers (TEI) **Status**: planned | **Priority**: medium | **Created**: 2026-07-12 diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index d0ad9238..aeb0274f 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -116,7 +116,7 @@ the backlog assumptions: - [x] Autouse fixture in `vektra-shared/tests/conftest.py` scrubs `VEKTRA_*` + external keys - [x] `make test` green with populated `.env` (638 passed) + `make lint` green - [x] Backlog entry updated (completed + resolution), changelog entry -- [ ] PR created and merged +- [x] PR created and merged (#84; #85 BUG-021, #86 backlog, #87 BUG-022 merged the same day) ### 2. Baseline eval (no branch; results recorded, not committed as code) - [x] Eval corpus verified intact in namespace `default` (excerpt corpus, 12 chunks — no reingest needed) @@ -126,19 +126,23 @@ the backlog assumptions: - [x] Numbers logged in this plan (Notes) and in vektra-internal (`stack/20260712-sprint3-baseline-eval.md`) ### 3. FEAT-017 — parent chunk expansion (branch `feat/feat-017-parent-chunk-expansion`) -- [ ] Propagate `parent_id` into `ChunkEmbedding` → Qdrant payload + pgvector column; - keep deterministic child/parent ids at store time -- [ ] Search excludes `chunk_level=parent` by default (both providers) -- [ ] `VEKTRA_PARENT_EXPANSION_ENABLED` (default false) in `QueryPipelineConfig` +- [x] Propagate `parent_id` into `ChunkEmbedding` → Qdrant payload + pgvector column; + keep deterministic child/parent ids at store time (pgvector now honors + caller-provided uuid5 ids instead of generating uuid4) +- [x] Search excludes `chunk_level=parent` by default (both providers) +- [x] `VEKTRA_PARENT_EXPANSION_ENABLED` (default false) in `QueryPipelineConfig` + mirrored in `VektraSettings` -- [ ] Expansion step in AdvancedQueryPipeline: fetch parent text by id via vector - store, replace child text **before** token budgeting (ARCH-055); dedup children - of the same parent (child text is a substring of parent → existing 80% overlap - dedup interacts) -- [ ] Trace metadata records expansion (children expanded, parents fetched) -- [ ] Reingest eval corpus with `dual` strategy; measure: dual+exclusion without - expansion (≈ fixed baseline expected), then with expansion; record both -- [ ] Unit tests: store-time linkage, search filter, expansion logic, budget accounting +- [x] Expansion step in AdvancedQueryPipeline: fetch parent text by id via new + `VectorStoreProvider.retrieve()`, replace child text **before** token + budgeting (ARCH-055); children of the same parent collapse into the + highest-scored one (runs after the 80% overlap dedup, so no interaction) +- [x] Trace metadata records expansion (children_expanded, siblings_merged, + parents_fetched) +- [x] Reingest eval corpus with `dual` strategy; measured both (see Notes): + A dual-no-expansion ≈ baseline on e2e but -6.5pp retrieval hit (boundary + effect); B expansion-on: zero e2e flips, sources 1.4 → 1.0 +- [x] Unit tests: store-time linkage, search filter, expansion logic, budget + accounting (17 new tests; suite 658 passed) ### 4. FEAT-018 — verification first (no branch unless justified) - [ ] Multi-turn scenario runner against `/api/v1/query` with `conversation_id` @@ -247,3 +251,30 @@ the bundle and a manual smoke in the Moodle dev stack. - March numbers (factual 90 / reasoning 80 / multi-chunk 10) are not comparable: different pipeline (pre BUG-015/016/017, pre FEAT-020) and per-question results were never versioned (gitignored file, since overwritten). +- 2026-07-12: **FEAT-017 measured** (full corpus `eval-full`, Combo D, same + dataset/config as baseline; corpus reingested with `dual` 500/100, parent 1500: + 105 points = 22 parents + 83 children, parents excluded from search). + Baseline to compare (fixed 500/100, 78 chunks): retrieval hit 89.1% / + MRR 0.8062 / P@5 0.3174; e2e grounded 35/55, multi-chunk 0/10, avg sources 1.4. + - **Measure A** (dual + parent exclusion, expansion off): retrieval hit 82.6% / + MRR 0.7029 / P@5 0.3130; e2e grounded 35/55 (64%), multi-chunk 1/10, avg + sources 1.4, p50 3935ms. Dual chunking itself costs ~6.5pp hit rate on this + corpus: children no longer roll overlap across parent boundaries, and 3 + questions whose keywords straddle a 1500-token section edge flip to miss + (IT-F-13, IT-R-02, EN-F-03). + - **Measure B** (expansion on): e2e grounded 35/55, multi-chunk 1/10, avg + sources 1.0 (sibling merge working: same grounding with a more compact + prompt), p50 3813ms. Zero per-question flips vs A. Expansion verified live + in traces (`parent_expansion` step, children_expanded/parents_fetched). + - **Root cause of the multi-chunk collapse found (and it is upstream of + FEAT-017)**: 9/10 multi-chunk questions end with `retrieval_filter + before=5 after=0` — bge-reranker-v2-m3 scores each partial-answer chunk + of a comparative/multi-part question below `min_relevance_score` 0.15 + (MC-01 with eval mode: max rerank score 0.088 while the raw RRF candidate + was 0.61). The whole candidate set is wiped before expansion can run. + Filed as TECH-007. Parent expansion works as designed but cannot touch + this failure mode; the Costituzione corpus also understates its benefit + (short self-contained articles — see TECH-005 collection 1). + - Local dev `.env` now: `VEKTRA_CHUNKING_STRATEGY=dual`, + `VEKTRA_PARENT_CHILD_LEVELS=1`, `VEKTRA_PARENT_EXPANSION_ENABLED=true`, + `VEKTRA_EVAL_MODE=true` (left on for FEAT-018 trace inspection). diff --git a/CHANGELOG.md b/CHANGELOG.md index f752c79d..06fe6a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,11 +19,20 @@ Convention (Keep a Changelog 1.1.0): +### Added + +- **rag**: optional parent chunk expansion in the advanced query pipeline (FEAT-017, `VEKTRA_PARENT_EXPANSION_ENABLED`, default off). With `VEKTRA_CHUNKING_STRATEGY=dual`, retrieved child chunks are replaced with their parent chunk's text after the retrieval filter and before token budgeting; children of the same parent collapse into the highest-scored one. Parent-child linkage is now actually persisted (deterministic `uuid5(doc_id, position)` ids, `parent_id` in the Qdrant payload and in the pgvector column), a new `VectorStoreProvider.retrieve()` fetches chunks by id, and the trace records `children_expanded`/`siblings_merged`/`parents_fetched` in a `parent_expansion` step. + +### Changed + +- **index**: vector search now excludes parent-level chunks (`chunk_level=parent`) in both providers; parents are context material fetched by id during expansion, not retrieval targets. Only affects documents ingested with `dual` chunking, whose parents previously polluted search results. +- **index**: pgvector `store()` honors caller-provided UUID chunk ids (deterministic ids from ingest) instead of always generating random ones; non-UUID ids still fall back to random. + ### Fixed - **docker**: the `INSTALL_UNSTRUCTURED=true` image variant builds again (BUG-022). torchvision (transitive via unstructured-inference) resolved from PyPI with CUDA-built wheels while torch is pinned to the CPU index, crashing the build with `operator torchvision::nms does not exist`. It is now declared in the `ocr` extra and pinned to the pytorch-cpu index; a new path-filtered CI workflow builds the OCR variant so it cannot silently regress. - **index**: `/api/v1/search` now resolves the embedding, sparse-embedding, and vector-store providers from the ProviderRegistry instead of hardcoding pgvector and reading a never-populated `app.state` attribute (BUG-021). In Qdrant deployments the endpoint returned zero results (it searched the empty `document_chunks` table) and hybrid mode always fell back to dense; the RAG pipeline (`/api/v1/query`) was unaffected. Found by the Sprint 3 baseline `make eval-retrieval` run. -- **tests**: unit tests are now hermetic against the developer's local `.env` (DEBT-025). Importing litellm during pytest collection loads `.env` into the process environment, which made 4 default-assertion tests in `vektra-shared` fail on dev machines while CI stayed green. An autouse fixture in `vektra-shared/tests/conftest.py` scrubs ambient `VEKTRA_*` variables; production settings loading is unchanged. +- **tests**: unit tests are now hermetic against the developer's local `.env` (DEBT-025). Importing litellm during pytest collection loads `.env` into the process environment, which made 4 default-assertion tests in `vektra-shared` fail on dev machines while CI stayed green. An autouse fixture in `vektra-shared/tests/conftest.py` scrubs ambient `VEKTRA_*` variables; production settings loading is unchanged. The fixture is replicated in `vektra-core` and `vektra-ingest` (FEAT-017 surfaced the same leak there: ambient `VEKTRA_CHUNKING_STRATEGY=dual` broke 12 chunker and pipeline-default tests). ## [0.5.1] - 2026-07-12 diff --git a/vektra-core/tests/conftest.py b/vektra-core/tests/conftest.py new file mode 100644 index 00000000..577e6fbb --- /dev/null +++ b/vektra-core/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared fixtures for vektra-core unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding config defaults (e.g. VEKTRA_PARENT_EXPANSION_ENABLED, + VEKTRA_EVAL_MODE). CI has no .env and never sees the difference. Tests + that need a specific value still set it explicitly via constructor + kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) diff --git a/vektra-ingest/tests/conftest.py b/vektra-ingest/tests/conftest.py new file mode 100644 index 00000000..7f4339f7 --- /dev/null +++ b/vektra-ingest/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared fixtures for vektra-ingest unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding config defaults (e.g. VEKTRA_CHUNKING_STRATEGY=dual makes + run_ingest pick the wrong chunker under test). CI has no .env and never + sees the difference. Tests that need a specific value still set it + explicitly via constructor kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) From 3eb25235708e74e223fad014b440c42dd0f7feae Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 21:54:08 +0000 Subject: [PATCH 17/31] fix(index): validate chunk ids before Qdrant retrieve; dedup via dict.fromkeys Addresses review comments 3567194789 (invalid point ids would fail the whole retrieve batch server-side; now filtered and logged like pgvector) and 3567194791 (O(N^2) unique-preserving scan). +2 tests. Co-Authored-By: Claude Fable 5 --- .../src/vektra_core/advanced_pipeline.py | 5 +--- .../src/vektra_index/providers/qdrant.py | 12 ++++++-- vektra-index/tests/test_qdrant_provider.py | 30 ++++++++++++++++--- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 1a8eea16..b328f34f 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -447,10 +447,7 @@ async def _expand_parents( (ARCH-055) so the budget sees the expanded text. Results keep the child's chunk_id and score for trace comparability. """ - parent_ids: list[str] = [] - for r in results: - if r.parent_id and r.parent_id not in parent_ids: - parent_ids.append(r.parent_id) + parent_ids = list(dict.fromkeys(r.parent_id for r in results if r.parent_id)) if not parent_ids: return results, {"children_expanded": 0, "parents_fetched": 0} diff --git a/vektra-index/src/vektra_index/providers/qdrant.py b/vektra-index/src/vektra_index/providers/qdrant.py index 9904dbef..a5028de9 100644 --- a/vektra-index/src/vektra_index/providers/qdrant.py +++ b/vektra-index/src/vektra_index/providers/qdrant.py @@ -345,12 +345,20 @@ async def retrieve( Points whose payload namespace does not match are dropped (namespace isolation): Qdrant retrieve() takes no filter. """ - if not chunk_ids: + valid_ids: list[str] = [] + for cid in chunk_ids: + try: + UUID(cid) + except ValueError: + logger.warning("qdrant_retrieve_invalid_chunk_id: %s", cid) + continue + valid_ids.append(cid) + if not valid_ids: return [] records = await self._client.retrieve( collection_name=self._collection_name, - ids=chunk_ids, + ids=valid_ids, with_payload=True, ) matching = [ diff --git a/vektra-index/tests/test_qdrant_provider.py b/vektra-index/tests/test_qdrant_provider.py index 0b5ffe16..16a60d76 100644 --- a/vektra-index/tests/test_qdrant_provider.py +++ b/vektra-index/tests/test_qdrant_provider.py @@ -329,11 +329,13 @@ async def test_retrieve_filters_by_namespace(self): provider, client = _make_provider() doc_id = str(uuid4()) + parent_id = str(uuid4()) + alien_id = str(uuid4()) # Qdrant retrieve() returns Record objects with no score attribute records = [ SimpleNamespace( - id="parent-1", + id=parent_id, payload={ "text": "parent text", "document_id": doc_id, @@ -343,7 +345,7 @@ async def test_retrieve_filters_by_namespace(self): }, ), SimpleNamespace( - id="alien-1", + id=alien_id, payload={ "text": "other tenant", "document_id": doc_id, @@ -355,13 +357,24 @@ async def test_retrieve_filters_by_namespace(self): ] client.retrieve = AsyncMock(return_value=records) - results = await provider.retrieve("default", ["parent-1", "alien-1"]) + results = await provider.retrieve("default", [parent_id, alien_id]) client.retrieve.assert_awaited_once() - assert [r.chunk_id for r in results] == ["parent-1"] + assert [r.chunk_id for r in results] == [parent_id] assert results[0].score == 0.0 assert results[0].text_snippet == "parent text" + @pytest.mark.asyncio + async def test_retrieve_skips_invalid_ids(self): + """Non-UUID ids never reach the Qdrant client (it would reject the batch).""" + provider, client = _make_provider() + client.retrieve = AsyncMock(return_value=[]) + valid = str(uuid4()) + + await provider.retrieve("default", ["not-a-uuid", valid]) + + assert client.retrieve.await_args.kwargs["ids"] == [valid] + @pytest.mark.asyncio async def test_retrieve_empty_ids_short_circuits(self): provider, client = _make_provider() @@ -370,3 +383,12 @@ async def test_retrieve_empty_ids_short_circuits(self): assert results == [] client.retrieve.assert_not_called() + + @pytest.mark.asyncio + async def test_retrieve_all_invalid_ids_short_circuits(self): + provider, client = _make_provider() + + results = await provider.retrieve("default", ["parent-1", "child-2"]) + + assert results == [] + client.retrieve.assert_not_called() From a479776c2e780916c44fea9b6542b910ae5a0547 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 22:03:40 +0000 Subject: [PATCH 18/31] docs(backlog): FEAT-018 verified no-go, deferred behind TECH-007 Four multi-turn scenarios (same-topic follow-up, topic switch, give-me-more, negation) run against /api/v1/query with conversation_id on eval-full. Rewrite + FEAT-020 history already mitigate topic switch and negation; same-topic follow-ups die at the rerank threshold (TECH-007 signature); "give me more" confirms the premise but degrades gracefully. Exclusion with the current funnel would increase refusals. Plan section 4 checked off. Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 11 ++++++++-- .s2s/plans/20260712-sprint3-rag-quality.md | 24 +++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index d048fd96..7f58e5c1 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -65,8 +65,15 @@ The LLM already has native conversational coherence: it sees the full history an ### FEAT-018: Exclude previously retrieved chunks in multi-turn conversations -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 -**Depends on**: evaluate BUG-020 option 1 (prompt fix) first - this may not be needed if the prompt change resolves multi-turn coherence. +**Status**: deferred (2026-07-12, verified) | **Priority**: low | **Created**: 2026-03-28 +**Depends on**: TECH-007 (rerank+threshold funnel) - with the current funnel, excluding previously seen chunks would increase multi-turn refusals, not variety. + +**Verification (2026-07-12, Sprint 3, plan `20260712-sprint3-rag-quality` section 4)**: 4 multi-turn scenarios run against `/api/v1/query` with `conversation_id` on the `eval-full` corpus (rewrite + FEAT-020 history + FEAT-017 expansion active, eval-mode traces inspected): +- **Topic switch** and **negation**: already fully mitigated - the rewriter produces clean standalone queries, retrieved chunks change completely (0 shared), reranker scores high (0.99 / 0.82). No exclusion needed. +- **Same-topic follow-up** ("quali limiti prevede l'art. 21?"): rewrite is correct but the turn dies at the retrieval filter (max rerank 0.093 < 0.15) - a TECH-007 failure, which chunk exclusion would make worse, not better. +- **"Give me more"**: the one genuine FEAT-018 case. Rewrite is explicit ("altri diritti... diversi da quelli gia' citati") yet the prompt receives the same 3 chunks as turn 1 (3/3 shared); the LLM degrades gracefully via history (acknowledges prior answer, avoids verbatim repetition) but cannot produce genuinely new content from identical material. + +**Decision**: no-go for Sprint 3. Revisit only after TECH-007 lands (a funnel that admits more chunks makes exclusion safe and useful), with "give me more" as the driving scenario and the original design below. **Context**: in multi-turn conversations, the vector search returns the same high-scoring chunks every turn, even when the user explicitly asks for "other" or "different" results. The query rewrite contextualizes the question but the retrieval still matches on semantic similarity, which favors the same chunks. diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index aeb0274f..a99f562d 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -145,12 +145,15 @@ the backlog assumptions: accounting (17 new tests; suite 658 passed) ### 4. FEAT-018 — verification first (no branch unless justified) -- [ ] Multi-turn scenario runner against `/api/v1/query` with `conversation_id` +- [x] Multi-turn scenario runner against `/api/v1/query` with `conversation_id` (same-topic follow-up, topic switch, "give me others", negation) -- [ ] Evaluate with FEAT-020 grounding modes: does history use already mitigate - the "same chunks every turn" complaint? -- [ ] Decision recorded in backlog (implement with Qdrant `must_not` + turn-chunk - tracking, or close as mitigated) — implementation only if tests justify it +- [x] Evaluate with FEAT-020 grounding modes: rewrite + history already mitigate + topic switch and negation fully (0 shared chunks, high rerank scores); + "give me more" is the one genuine case (3/3 identical chunks, graceful + degradation via history); same-topic follow-up dies at the threshold + (TECH-007), which exclusion would worsen +- [x] Decision recorded in backlog: **no-go for Sprint 3, deferred behind + TECH-007** — evidence in the FEAT-018 backlog entry and Notes below ### 5. FEAT-021 — per-namespace citations (branch `feat/feat-021-namespace-citations`) - [ ] `resolve_citations_enabled` in `vektra_shared/namespace.py` (clone of FEAT-020 @@ -278,3 +281,14 @@ the bundle and a manual smoke in the Moodle dev stack. - Local dev `.env` now: `VEKTRA_CHUNKING_STRATEGY=dual`, `VEKTRA_PARENT_CHILD_LEVELS=1`, `VEKTRA_PARENT_EXPANSION_ENABLED=true`, `VEKTRA_EVAL_MODE=true` (left on for FEAT-018 trace inspection). +- 2026-07-12: **FEAT-018 verified — no-go, deferred behind TECH-007.** Four + multi-turn scenarios against `/api/v1/query` with `conversation_id` + (eval-full corpus, expansion on, eval-mode traces): topic switch and negation + fully mitigated by rewrite (ARCH-061) + history (FEAT-020) — chunks change + completely, rerank 0.99/0.82; same-topic follow-up ("quali limiti prevede + l'art. 21?") rewrites correctly but dies at the retrieval filter (max rerank + 0.093 < 0.15 — same TECH-007 signature as multi-chunk); "give me more" + confirms the FEAT-018 premise (3/3 identical chunks despite an explicit + "diversi da quelli già citati" rewrite) but degrades gracefully via history. + Chunk exclusion today would raise refusals on follow-ups; revisit after + TECH-007. Full per-turn evidence in the FEAT-018 backlog entry. From bc629a66888d5ec4442b722e322298a6cbca3caf Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 22:19:46 +0000 Subject: [PATCH 19/31] feat(rag): per-namespace inline source citations (FEAT-021) New namespaces.config key citations_enabled (admin PATCH whitelist + GET resolved defaults, strictly bool, no env var, default off). When enabled, system.j2 Rule 1 instructs inline [n] markers matching the elements, context.j2 adds a title attribute ("filename, p.N" composed at prompt-build time from _fetch_document_names + chunk metadata page), and SourceRef/source payloads carry a title field (core sync + streaming, learn course response). The learn widget renders [n] as a superscript and attaches the source title as a tooltip in addSources(); bundle is built by the Docker widget-builder stage. Default-off renders byte-identical prompts; the advanced pipeline is the only consumer (simple pipeline untouched). Co-Authored-By: Claude Fable 5 --- vektra-admin/src/vektra_admin/api.py | 3 + .../src/vektra_core/advanced_pipeline.py | 72 +++++++++++++++++-- vektra-core/src/vektra_core/api.py | 9 ++- vektra-core/src/vektra_core/templates.py | 7 +- .../src/vektra_core/templates/context.j2 | 2 +- .../src/vektra_core/templates/system.j2 | 8 +++ vektra-learn/src/vektra_learn/api.py | 12 +++- vektra-learn/src/vektra_learn/query.py | 1 + vektra-learn/widget/src/chat-ui.js | 13 ++++ vektra-learn/widget/src/markdown.js | 7 ++ vektra-learn/widget/src/styles.js | 10 +++ vektra-shared/src/vektra_shared/namespace.py | 33 +++++++++ vektra-shared/src/vektra_shared/types.py | 5 ++ 13 files changed, 170 insertions(+), 12 deletions(-) diff --git a/vektra-admin/src/vektra_admin/api.py b/vektra-admin/src/vektra_admin/api.py index aa7ae1b5..a98caa24 100644 --- a/vektra-admin/src/vektra_admin/api.py +++ b/vektra-admin/src/vektra_admin/api.py @@ -63,6 +63,7 @@ } ALLOWED_CONFIG_TYPES: dict[str, type] = { "show_sources": bool, + "citations_enabled": bool, } assert ALLOWED_CONFIG_VALUES.keys().isdisjoint(ALLOWED_CONFIG_TYPES.keys()), ( "Each config key must appear in exactly one of " @@ -660,6 +661,8 @@ async def get_namespace_config( "show_sources": getattr( request.app.state, "learn_show_sources_default", True ), + # FEAT-021: no env var, hardcoded default off + "citations_enabled": False, }, ) return NamespaceConfigDetailResponse( diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index b328f34f..82aec7d4 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -484,11 +484,30 @@ async def _expand_parents( "after_expansion": len(expanded), } + @staticmethod + def _chunk_title( + result: SearchResult, + name_map: dict[str, str], + ) -> str | None: + """Human-readable source title for citations (FEAT-021). + + Prefers "document_name, p.N"; falls back to the raw source_file + from chunk metadata, then None (attribute omitted in the template). + """ + name = name_map.get(str(result.document_id)) or result.metadata.get( + "source_file" + ) + if not name: + return None + page = result.metadata.get("page") + return f"{name}, p.{page}" if page is not None else str(name) + def _build_prompt( self, query: QueryRequest, filtered: list[SearchResult], history: list[dict[str, str | None]], + name_map: dict[str, str] | None = None, ) -> tuple[ list[Message], list[SearchResult], list[dict[str, str | None]], StepTrace ]: @@ -500,6 +519,7 @@ def _build_prompt( namespace=query.namespace, grounding_mode=query.grounding_mode, has_context=has_context, + citations_enabled=query.citations_enabled, ) system_tokens = self._count_tokens(system_text) question_tokens = self._count_tokens(query.question) @@ -525,8 +545,20 @@ def _build_prompt( selected_history = [history[i] for i in selected_history_idx] if selected_chunks: + _names = name_map or {} context_text = self._renderer.render_context( - [{"text": r.text_snippet, "score": r.score} for r in selected_chunks] + [ + { + "text": r.text_snippet, + "score": r.score, + "title": ( + self._chunk_title(r, _names) + if query.citations_enabled + else None + ), + } + for r in selected_chunks + ] ) user_content = f"{context_text}\n\nQuestion: {query.question}" else: @@ -606,15 +638,25 @@ async def execute( no_relevant_context=no_relevant_context and not safeguard_blocked, ), trace - # Step 7: Build prompt + # Step 7: Build prompt (citations need document names up front, FEAT-021) + prompt_name_map: dict[str, str] = {} + if query.citations_enabled and filtered: + prompt_name_map = await _fetch_document_names( + [r.document_id for r in filtered] + ) messages, selected_chunks, _, prompt_step = self._build_prompt( - query, filtered, history + query, filtered, history, name_map=prompt_name_map ) steps.append(prompt_step) # Sources from budget-selected chunks only (not all filtered) t0 = time.monotonic() - name_map = await _fetch_document_names([r.document_id for r in selected_chunks]) + if query.citations_enabled: + name_map = prompt_name_map + else: + name_map = await _fetch_document_names( + [r.document_id for r in selected_chunks] + ) steps.append( StepTrace( name="document_names", @@ -634,6 +676,9 @@ async def execute( citation_id=uuid4(), document_version=r.document_version, document_name=name_map.get(str(r.document_id)), + title=( + self._chunk_title(r, name_map) if query.citations_enabled else None + ), ) for r in selected_chunks ] @@ -773,9 +818,14 @@ async def _stream( yield QueryChunk(type="done", data="") return - # Step 7: Build prompt + # Step 7: Build prompt (citations need document names up front, FEAT-021) + prompt_name_map: dict[str, str] = {} + if query.citations_enabled and filtered: + prompt_name_map = await _fetch_document_names( + [r.document_id for r in filtered] + ) messages, selected_chunks, _, prompt_step = self._build_prompt( - query, filtered, history + query, filtered, history, name_map=prompt_name_map ) steps.append(prompt_step) @@ -874,7 +924,12 @@ async def _stream( # Yield sources (only budget-selected chunks, not all filtered) t0 = time.monotonic() - name_map = await _fetch_document_names([r.document_id for r in selected_chunks]) + if query.citations_enabled: + name_map = prompt_name_map + else: + name_map = await _fetch_document_names( + [r.document_id for r in selected_chunks] + ) steps.append( StepTrace( name="document_names", @@ -894,6 +949,9 @@ async def _stream( "citation_id": str(uuid4()), "document_version": r.document_version, "document_name": name_map.get(str(r.document_id)), + "title": ( + self._chunk_title(r, name_map) if query.citations_enabled else None + ), } for r in selected_chunks ] diff --git a/vektra-core/src/vektra_core/api.py b/vektra-core/src/vektra_core/api.py index 023c563d..960831ea 100644 --- a/vektra-core/src/vektra_core/api.py +++ b/vektra-core/src/vektra_core/api.py @@ -38,7 +38,7 @@ ErrorResponse, http_status_for, ) -from vektra_shared.namespace import resolve_grounding_mode +from vektra_shared.namespace import resolve_citations_enabled, resolve_grounding_mode from vektra_shared.types import ( QueryChunk, QueryRequest, @@ -77,6 +77,7 @@ class SourceRefBody(BaseModel): citation_id: UUID document_version: int = 1 document_name: str | None = None # FEAT-012 + title: str | None = None # FEAT-021: set when the namespace cites sources class QueryResponseBody(BaseModel): @@ -270,6 +271,11 @@ async def query( else: grounding_mode = _default_mode + # Resolve citations: namespace config > default false (FEAT-021) + citations_enabled = False + if db_factory: + citations_enabled = await resolve_citations_enabled(body.namespace, db_factory) + query_req = QueryRequest( question=body.question, namespace=body.namespace, @@ -277,6 +283,7 @@ async def query( top_k=body.top_k, stream=use_stream, grounding_mode=grounding_mode, + citations_enabled=citations_enabled, ) if use_stream: diff --git a/vektra-core/src/vektra_core/templates.py b/vektra-core/src/vektra_core/templates.py index e6db0304..21e2ede4 100644 --- a/vektra-core/src/vektra_core/templates.py +++ b/vektra-core/src/vektra_core/templates.py @@ -95,16 +95,19 @@ def render_system( namespace: str = "default", grounding_mode: str = "strict", has_context: bool = True, + citations_enabled: bool = False, ) -> str: - """Render system.j2 with namespace, grounding mode, and context flag.""" + """Render system.j2 with namespace, grounding mode, and context/citation flags.""" tmpl = self._env.get_template("system.j2") return tmpl.render( namespace=namespace, grounding_mode=grounding_mode, has_context=has_context, + citations_enabled=citations_enabled, ) def render_context(self, chunks: list[dict[str, Any]]) -> str: - """Render context.j2 with a list of chunk dicts containing 'text' (and optionally 'score').""" + """Render context.j2 with a list of chunk dicts containing 'text' and + optionally 'score' and 'title' (source title attribute, FEAT-021).""" tmpl = self._env.get_template("context.j2") return tmpl.render(chunks=chunks) diff --git a/vektra-core/src/vektra_core/templates/context.j2 b/vektra-core/src/vektra_core/templates/context.j2 index cde635de..7a0364dc 100644 --- a/vektra-core/src/vektra_core/templates/context.j2 +++ b/vektra-core/src/vektra_core/templates/context.j2 @@ -1,3 +1,3 @@ -{% for chunk in chunks %}{{ chunk.text | e }} +{% for chunk in chunks %}{{ chunk.text | e }} {% endfor %} diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index 5eaf1dee..ed43925e 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -30,8 +30,16 @@ enough information to answer. {% endif %} {% endif %} Rules: +{% if citations_enabled and has_context %} +1. Cite the sources you used with inline [id] markers matching the id + attributes of the elements, placed at the end of the sentence + they support. If multiple sources support a claim, list them together, + e.g. [1][3]. Do not quote source titles or talk about the retrieval + process; the markers are the only visible reference. +{% else %} 1. Sound like you simply know the answer. Never mention, quote, or allude to sources, documents, context tags, or reference material. +{% endif %} 2. Never offer to search, look up, or provide more information later. 3. If a source is cut off, use what is available without commenting on it. 4. Respond in the same language the user writes in. diff --git a/vektra-learn/src/vektra_learn/api.py b/vektra-learn/src/vektra_learn/api.py index a5ee5ca9..16b77ef6 100644 --- a/vektra-learn/src/vektra_learn/api.py +++ b/vektra-learn/src/vektra_learn/api.py @@ -51,7 +51,11 @@ ErrorResponse, http_status_for, ) -from vektra_shared.namespace import resolve_grounding_mode, resolve_show_sources +from vektra_shared.namespace import ( + resolve_citations_enabled, + resolve_grounding_mode, + resolve_show_sources, +) from vektra_shared.types import trace_from_dict # Sentinel key_id for learn-originated conversations (JWT auth has no API key). @@ -777,6 +781,12 @@ async def course_query( else: _show_sources = _default_show_sources + # Resolve citations: namespace config > default false (FEAT-021) + if _db_factory_gm: + query_req.citations_enabled = await resolve_citations_enabled( + namespace, _db_factory_gm + ) + if registry is None: err = ErrorResponse( category=ErrorCategory.TRANSIENT, diff --git a/vektra-learn/src/vektra_learn/query.py b/vektra-learn/src/vektra_learn/query.py index 6762d63c..e9d3e92e 100644 --- a/vektra-learn/src/vektra_learn/query.py +++ b/vektra-learn/src/vektra_learn/query.py @@ -81,6 +81,7 @@ def pipeline_response_to_course_response( "score": s.score, "snippet": s.snippet, "document_name": s.document_name, # FEAT-012 + "title": s.title, # FEAT-021: citation tooltip label } for s in resp.sources ], diff --git a/vektra-learn/widget/src/chat-ui.js b/vektra-learn/widget/src/chat-ui.js index f2291075..a5f37574 100644 --- a/vektra-learn/widget/src/chat-ui.js +++ b/vektra-learn/widget/src/chat-ui.js @@ -408,6 +408,19 @@ export class ChatUI { const msgEl = msgOrStream.el || msgOrStream; if (!sources || sources.length === 0) return; + // FEAT-021: link [n] citation markers in the answer to their source. + // Marker order matches the sources order (both follow the prompt's + // budget-selected chunk order), so [n] -> sources[n-1]. + for (const cite of msgEl.querySelectorAll("sup.vektra-cite")) { + const idx = parseInt(cite.getAttribute("data-cite"), 10) - 1; + const src = sources[idx]; + const label = src && (src.title || src.document_name); + if (label) { + cite.title = label; + cite.classList.add("linked"); + } + } + const container = document.createElement("div"); container.className = "vektra-chat-sources"; diff --git a/vektra-learn/widget/src/markdown.js b/vektra-learn/widget/src/markdown.js index 960e2ca3..7375367d 100644 --- a/vektra-learn/widget/src/markdown.js +++ b/vektra-learn/widget/src/markdown.js @@ -150,5 +150,12 @@ function renderInline(text) { } return `${label} (${url})`; }) + // Citation markers [n] (FEAT-021): must run after links so [label](url) + // is already consumed. addSources() attaches the source title as a + // tooltip when the namespace has citations enabled. + .replace( + /\[(\d{1,2})\]/g, + '[$1]', + ) ); } diff --git a/vektra-learn/widget/src/styles.js b/vektra-learn/widget/src/styles.js index 944c7a3f..7696fc2d 100644 --- a/vektra-learn/widget/src/styles.js +++ b/vektra-learn/widget/src/styles.js @@ -209,6 +209,16 @@ export function buildStyles(theme) { font-weight: 600; } +.vektra-chat-msg sup.vektra-cite { + font-size: 0.75em; + vertical-align: super; + line-height: 1; +} +.vektra-chat-msg sup.vektra-cite.linked { + color: var(--vektra-primary, ${t.primary}); + cursor: help; + font-weight: 600; +} .vektra-chat-sources { margin-top: 8px; padding-top: 8px; diff --git a/vektra-shared/src/vektra_shared/namespace.py b/vektra-shared/src/vektra_shared/namespace.py index 5229be6e..8fa4a53e 100644 --- a/vektra-shared/src/vektra_shared/namespace.py +++ b/vektra-shared/src/vektra_shared/namespace.py @@ -49,6 +49,39 @@ async def resolve_grounding_mode( return default_mode +async def resolve_citations_enabled( + namespace: str, + session_factory: Any, + default_value: bool = False, +) -> bool: + """Resolve citations_enabled: namespace JSONB config > default (FEAT-021). + + There is no env var for this setting: citations are a per-namespace + deployment choice (academic/compliance contexts), default off. + + Returns *default_value* on any error (namespace not found, DB error, + non-boolean value in config). + """ + try: + async with session_factory() as session: + result = await session.execute( + text("SELECT config FROM namespaces WHERE id = :ns"), + {"ns": namespace}, + ) + row = result.scalar_one_or_none() + if row and isinstance(row, dict): + ns_value = row.get("citations_enabled") + if isinstance(ns_value, bool): + return ns_value + except Exception as exc: + log.debug( + "citations_enabled_resolution_fallback", + namespace=namespace, + error=str(exc), + ) + return default_value + + async def resolve_show_sources( namespace: str, session_factory: Any, diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index 38729007..4cc711a1 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -273,6 +273,7 @@ class QueryRequest: filters: SearchFilters | None = None stream: bool = False grounding_mode: str = "strict" # "strict" | "hybrid" (FEAT-020) + citations_enabled: bool = False # per-namespace inline citations (FEAT-021) @dataclass @@ -290,6 +291,10 @@ class SourceRef: # documents (REQ-057) appear with an " (archived)" suffix; None only when # the DB lookup failed (transient error, DB not initialised in tests). document_name: str | None = None + # FEAT-021: human-readable source title ("filename, p.N") matching the + # title attribute rendered in the prompt context; set only when the + # namespace has citations_enabled. + title: str | None = None @dataclass From 49a7cee18ed7c99dd698fdec74fadb62d7bb5b59 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 22:19:56 +0000 Subject: [PATCH 20/31] test(rag): cover citations resolution, templates, pipeline flow (FEAT-021) - shared: resolve_citations_enabled (config true, missing key, non-bool value ignored, db error -> default false) - core templates: Rule 1 both branches, no-citations-without-context, default-off byte-identical rendering, title attribute escaped and omitted when absent, legacy chunks without title key unchanged - core pipeline: citations flow end to end (system rule, context title, SourceRef.title) and default-off leaves prompt untouched - admin integration: PATCH sets citations_enabled, rejects non-bool, GET resolved includes the new key with default false Co-Authored-By: Claude Fable 5 --- vektra-admin/tests/test_integration.py | 51 ++++++++++++++- vektra-core/tests/test_advanced_pipeline.py | 70 +++++++++++++++++++++ vektra-core/tests/test_templates.py | 58 +++++++++++++++++ vektra-shared/tests/test_namespace.py | 67 +++++++++++++++++++- 4 files changed, 244 insertions(+), 2 deletions(-) diff --git a/vektra-admin/tests/test_integration.py b/vektra-admin/tests/test_integration.py index 50c550bf..86df7aef 100644 --- a/vektra-admin/tests/test_integration.py +++ b/vektra-admin/tests/test_integration.py @@ -912,9 +912,14 @@ async def test_namespace_config_get_returns_empty_with_resolved_defaults( # Hardcoded resolver defaults: grounding_mode="strict", show_sources=True. # In CI the env vars may shift these; assert the keys are present and bool/string-typed # rather than a fixed value, so the test stays independent of test env config. - assert set(body["resolved"]) == {"grounding_mode", "show_sources"} + assert set(body["resolved"]) == { + "grounding_mode", + "show_sources", + "citations_enabled", + } assert body["resolved"]["grounding_mode"] in {"strict", "hybrid"} assert isinstance(body["resolved"]["show_sources"], bool) + assert body["resolved"]["citations_enabled"] is False # FEAT-021 default async def test_namespace_config_get_reflects_stored_overrides( @@ -981,3 +986,47 @@ async def test_namespace_config_get_requires_admin_scope( headers={"Authorization": f"Bearer {query_key}"}, ) assert resp.status_code == 403, resp.text + + +# --------------------------------------------------------------------------- +# FEAT-021: citations_enabled per-namespace setting +# --------------------------------------------------------------------------- + + +async def test_namespace_config_patch_sets_citations_enabled( + client, bootstrap_key, fresh_engine +): + """PATCH with citations_enabled=true persists as a bool in namespaces.config.""" + admin_key = await _create_admin_key(client, bootstrap_key) + ns_id = "feat21-citations-true" + await _seed_namespace(fresh_engine, ns_id) + + resp = await client.patch( + f"/api/v1/admin/namespaces/{ns_id}/config", + json={"citations_enabled": True}, + headers={"Authorization": f"Bearer {admin_key}"}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["config"] == {"citations_enabled": True} + + await asyncio.sleep(0.2) + stored = await _read_namespace_config(fresh_engine, ns_id) + assert stored == {"citations_enabled": True} + + +async def test_namespace_config_patch_rejects_citations_enabled_non_bool( + client, bootstrap_key, fresh_engine +): + """citations_enabled is strictly bool: strings are rejected with ERR-ADMIN-007.""" + admin_key = await _create_admin_key(client, bootstrap_key) + ns_id = "feat21-citations-nonbool" + await _seed_namespace(fresh_engine, ns_id) + + resp = await client.patch( + f"/api/v1/admin/namespaces/{ns_id}/config", + json={"citations_enabled": "yes"}, + headers={"Authorization": f"Bearer {admin_key}"}, + ) + assert resp.status_code == 400, resp.text + err = resp.json()["detail"]["error"] + assert err["code"] == "ERR-ADMIN-007" diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index 2956340c..249da06b 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -959,3 +959,73 @@ def _count(text: str, model: str | None = None) -> int: # The budget counted the parent text, not the child snippet assert any(parent_text == t for t in token_counts) assert all(t != "tiny" for t in token_counts) + + +# --------------------------------------------------------------------------- +# Per-namespace citations (FEAT-021) +# --------------------------------------------------------------------------- + + +async def test_citations_enabled_flows_to_prompt_and_sources(): + """citations_enabled=True: Rule 1 swaps, context gets titles, SourceRef.title set.""" + from unittest.mock import patch + from uuid import UUID as _UUID + + doc_id = uuid4() + result = SearchResult( + chunk_id=str(uuid4()), + score=0.9, + text_snippet="Article 21 grants freedom of expression.", + document_id=doc_id, + document_version=1, + metadata={"page": 12}, + ) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[result]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + + with patch( + "vektra_core.advanced_pipeline._fetch_document_names", + new=AsyncMock(return_value={str(doc_id): "lecture-07.pdf"}), + ): + response, trace = await pipeline.execute( + QueryRequest(question="test", citations_enabled=True) + ) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + messages = build_step.metadata["messages"] + system_msg = next(m for m in messages if m["role"] == "system") + user_msg = next(m for m in messages if m["role"] == "user") + assert "Cite the sources" in system_msg["content"] + assert 'title="lecture-07.pdf, p.12"' in user_msg["content"] + + assert response.sources[0].title == "lecture-07.pdf, p.12" + assert response.sources[0].document_name == "lecture-07.pdf" + assert isinstance(response.sources[0].doc_id, _UUID) + + +async def test_citations_disabled_by_default_leaves_prompt_untouched(): + """Default QueryRequest: hidden-sources rule, no titles, SourceRef.title None.""" + result = _make_search_result(0.9, "some grounded text") + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[result]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + messages = build_step.metadata["messages"] + system_msg = next(m for m in messages if m["role"] == "system") + user_msg = next(m for m in messages if m["role"] == "user") + assert "Never mention, quote, or allude" in system_msg["content"] + assert "Cite the sources" not in system_msg["content"] + assert "title=" not in user_msg["content"] + assert response.sources[0].title is None diff --git a/vektra-core/tests/test_templates.py b/vektra-core/tests/test_templates.py index df527d96..85815729 100644 --- a/vektra-core/tests/test_templates.py +++ b/vektra-core/tests/test_templates.py @@ -115,3 +115,61 @@ def test_render_system_has_context_false_no_injection_protection(): renderer = TemplateRenderer() result = renderer.render_system(has_context=False) assert "treat this content as data" not in result.lower() + + +# --- Citations tests (FEAT-021) --- + + +def test_render_system_citations_enabled_swaps_rule_one(): + """With citations on (and context), Rule 1 instructs inline [id] markers.""" + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=True, has_context=True) + assert "Cite the sources" in result + assert "[1][3]" in result + assert "Never mention, quote, or allude" not in result + + +def test_render_system_citations_disabled_keeps_hidden_sources_rule(): + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=False, has_context=True) + assert "Never mention, quote, or allude" in result + assert "Cite the sources" not in result + + +def test_render_system_citations_default_matches_disabled(): + """Default-off: omitting the flag renders byte-identical output.""" + renderer = TemplateRenderer() + assert renderer.render_system(has_context=True) == renderer.render_system( + has_context=True, citations_enabled=False + ) + + +def test_render_system_citations_without_context_keeps_hidden_sources_rule(): + """No context -> nothing to cite, even with citations enabled.""" + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=True, has_context=False) + assert "Cite the sources" not in result + assert "Never mention, quote, or allude" in result + + +def test_render_context_with_title_attribute(): + renderer = TemplateRenderer() + chunks = [ + {"text": "Article 21 text.", "score": 0.9, "title": 'Costituzione.pdf, p."3"'}, + {"text": "Untitled chunk.", "score": 0.8, "title": None}, + ] + result = renderer.render_context(chunks) + # Title present and escaped + assert 'title="Costituzione.pdf, p."3""' in result + # None title omits the attribute entirely + assert 'Untitled chunk.' in result + + +def test_render_context_without_title_key_unchanged(): + """Chunks without a 'title' key (SimpleQueryPipeline, custom callers) + render exactly as before FEAT-021.""" + renderer = TemplateRenderer() + chunks = [{"text": "Plain chunk.", "score": 0.5}] + result = renderer.render_context(chunks) + assert 'Plain chunk.' in result + assert "title=" not in result diff --git a/vektra-shared/tests/test_namespace.py b/vektra-shared/tests/test_namespace.py index 7dc03c81..163f8468 100644 --- a/vektra-shared/tests/test_namespace.py +++ b/vektra-shared/tests/test_namespace.py @@ -4,7 +4,10 @@ import pytest -from vektra_shared.namespace import resolve_grounding_mode +from vektra_shared.namespace import ( + resolve_citations_enabled, + resolve_grounding_mode, +) @pytest.mark.asyncio @@ -92,3 +95,65 @@ async def test_returns_default_for_invalid_mode_in_config(): "test-ns", mock_factory, default_mode="strict" ) assert result == "strict" + + +# --- citations_enabled tests (FEAT-021) --- + + +@pytest.mark.asyncio +async def test_citations_enabled_from_namespace_config(): + """Namespace config.citations_enabled=true overrides the default.""" + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"citations_enabled": True} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is True + + +@pytest.mark.asyncio +async def test_citations_enabled_defaults_false_when_key_missing(): + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"grounding_mode": "strict"} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False + + +@pytest.mark.asyncio +async def test_citations_enabled_ignores_non_bool_value(): + """A truthy non-bool ("yes", 1) in config must not enable citations.""" + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"citations_enabled": "yes"} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False + + +@pytest.mark.asyncio +async def test_citations_enabled_defaults_false_on_db_error(): + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock( + side_effect=RuntimeError("db down") + ) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False From bf5d9b2cfbc5b0ef8b257dbddf8cc7568e498837 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 22:19:56 +0000 Subject: [PATCH 21/31] docs: FEAT-021 completed (backlog, plan section 5, changelog, api.md) Backlog resolution records the deltas vs the original design ( tag kept, no new SearchResult fields, no env var). api.md documents the citations_enabled config key and the sources[].title response field. Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 17 +++++++------ .s2s/plans/20260712-sprint3-rag-quality.md | 29 ++++++++++++++-------- CHANGELOG.md | 1 + docs/reference/api.md | 3 +++ 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index d048fd96..bff42262 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -223,7 +223,8 @@ Rules: ### FEAT-021: Optional source citations in responses (per-namespace) -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 +**Status**: completed (2026-07-12) | **Priority**: medium | **Created**: 2026-03-28 +**Resolution**: shipped on `feat/feat-021-namespace-citations` (Sprint 3, plan `20260712-sprint3-rag-quality` section 5). Deltas vs the design below: the context template keeps the existing `` tag (the `` sketch predates the template); `SearchResult` gains no new fields (source_file/page were already in `metadata`; the title is composed at prompt-build time from `_fetch_document_names` + page); resolution is namespace JSONB > hardcoded false, no env var; the widget renders `[n]` as superscript with the source title as native tooltip, wired in `addSources()`. Default-off renders byte-identical prompts (`prompt_version` changes because the template files changed). **Context**: in some deployment contexts (academic research, compliance, legal), full transparency with source citations is required. Currently Rule 1 in the system prompt forbids any mention of sources ("Never mention, quote, or allude to sources, documents, context tags, or reference material"). This is correct for the default e-learning use case where the student should not know about the RAG pipeline, but must be optional for contexts where traceability is a requirement. @@ -271,13 +272,13 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. **Traceability**: ARCH-054 (composable templates), ARCH-047 (namespace metadata), ADR-0025 (chatbot widget) **Acceptance criteria**: -- [ ] `citations_enabled` per-namespace setting in namespace metadata JSONB -- [ ] `system.j2` Rule 1 conditional: cite with `[id]` when enabled, hide sources when disabled -- [ ] `context.j2` includes document title in `` elements when citations enabled -- [ ] Document filename and page propagated through `SearchResult` to template -- [ ] `SourceRef` includes `title` field in API response -- [ ] Widget renders `[id]` references as tooltips or footnotes with source info -- [ ] Default behavior unchanged (citations disabled, Rule 1 hides sources) +- [x] `citations_enabled` per-namespace setting in namespace metadata JSONB (admin PATCH whitelist + GET resolved defaults) +- [x] `system.j2` Rule 1 conditional: cite with `[id]` when enabled (and context present), hide sources when disabled +- [x] `context.j2` includes document title in `` elements when citations enabled +- [x] Document filename and page propagated to the template (composed at prompt-build time from `_fetch_document_names` + `metadata.page`) +- [x] `SourceRef` includes `title` field in API response (core + learn, sync + streaming) +- [x] Widget renders `[id]` references as superscripts with source-title tooltips +- [x] Default behavior unchanged (citations disabled, Rule 1 hides sources; byte-identical rendering asserted in tests) --- diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index aeb0274f..0fab50cd 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -153,16 +153,25 @@ the backlog assumptions: tracking, or close as mitigated) — implementation only if tests justify it ### 5. FEAT-021 — per-namespace citations (branch `feat/feat-021-namespace-citations`) -- [ ] `resolve_citations_enabled` in `vektra_shared/namespace.py` (clone of FEAT-020 - pattern), default false; admin config PATCH whitelist extended -- [ ] `QueryRequest.citations_enabled` resolved at API layer, passed to renderer -- [ ] `system.j2` Rule 1 conditional (cite `[id]` inline when enabled) -- [ ] `context.j2`: add `title` attribute (document_name + page) when enabled -- [ ] Propagate `source_file`/`page`/`title` through to `SourceRef` (API response) -- [ ] Widget: render inline `[n]` markers as interactive references (tooltip/footnote - with source title + snippet); esbuild bundle rebuilt -- [ ] Default-off behavior byte-identical prompts (prompt_version changes documented) -- [ ] Unit tests: namespace resolution, both template branches, SourceRef fields +- [x] `resolve_citations_enabled` in `vektra_shared/namespace.py` (clone of FEAT-020 + pattern), default false; admin config PATCH whitelist + GET resolved defaults + extended +- [x] `QueryRequest.citations_enabled` resolved at API layer (core + learn), passed + to renderer +- [x] `system.j2` Rule 1 conditional (cite `[id]` inline when enabled and context + present) +- [x] `context.j2`: `title` attribute (document_name + page, composed at + prompt-build time) when enabled; template tolerant of chunks without title +- [x] `SourceRef.title` in API responses (core sync + streaming, learn course + response); `source_file`/`page` were already in `SearchResult.metadata` +- [x] Widget: `[n]` markers rendered as superscripts with source-title tooltip + wired in `addSources()`; bundle built by the Docker widget-builder stage + (static/ is gitignored) +- [x] Default-off behavior byte-identical prompts, asserted in template tests + (prompt_version changes because template files changed — documented in + changelog) +- [x] Unit tests: namespace resolution (4), templates (6), pipeline flow (2), + admin integration (2 + resolved defaults); suite 672 passed ## State & Data Lifecycle diff --git a/CHANGELOG.md b/CHANGELOG.md index 06fe6a24..cd842679 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Convention (Keep a Changelog 1.1.0): ### Added +- **rag**: optional per-namespace inline source citations (FEAT-021, `citations_enabled` in the namespace config JSONB via `PATCH /api/v1/admin/namespaces/{id}/config`, default off, advanced pipeline only). When enabled, the system prompt instructs the LLM to add inline `[n]` markers matching the `` elements, the context template carries a `title` attribute ("filename, p.N"), and each returned source includes a `title` field; the learn widget renders the markers as superscripts with a tooltip. Default-off renders byte-identical prompts; `prompt_version` changes anyway because the template files changed (trace comparability note). - **rag**: optional parent chunk expansion in the advanced query pipeline (FEAT-017, `VEKTRA_PARENT_EXPANSION_ENABLED`, default off). With `VEKTRA_CHUNKING_STRATEGY=dual`, retrieved child chunks are replaced with their parent chunk's text after the retrieval filter and before token budgeting; children of the same parent collapse into the highest-scored one. Parent-child linkage is now actually persisted (deterministic `uuid5(doc_id, position)` ids, `parent_id` in the Qdrant payload and in the pgvector column), a new `VectorStoreProvider.retrieve()` fetches chunks by id, and the trace records `children_expanded`/`siblings_merged`/`parents_fetched` in a `parent_expansion` step. ### Changed diff --git a/docs/reference/api.md b/docs/reference/api.md index 99dd9ba5..d1d75611 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -194,6 +194,7 @@ Request body (flat dict, one entry per config key): |-------|------|----------------|-------------| | `grounding_mode` | string or null | `"strict"`, `"hybrid"`, `null` | RAG grounding policy. `null` removes the key and falls back to `VEKTRA_PROMPT_GROUNDING_MODE`. | | `show_sources` | bool or null | `true`, `false`, `null` | Widget citation visibility (FEAT-014). `null` removes the key and falls back to `VEKTRA_LEARN_SHOW_SOURCES`. The API always returns the full sources list; the flag only instructs the widget whether to render them. Resolution chain: client `data-show-sources` attr > `namespaces.config.show_sources` > `VEKTRA_LEARN_SHOW_SOURCES` env > hardcoded `true`. | +| `citations_enabled` | bool or null | `true`, `false`, `null` | Inline source citations (FEAT-021, advanced pipeline only). When `true`, the LLM is instructed to add `[n]` markers matching the context sources, and each returned source carries a `title` ("filename, p.N") for tooltip rendering. No env var: `null` (or absent) means disabled. Default-off leaves prompts unchanged. | Behavior: - **Partial update**: keys not present in the body are preserved. @@ -340,6 +341,7 @@ Response: | `answer` | LLM-generated answer grounded in sources | | `sources` | Ranked list of source chunks | | `sources[].document_name` | Filename of the source document (e.g. `lecture-07.pdf`), or `null` when the document join returns no row. Soft-deleted documents (REQ-057) keep their citation with an `(archived)` suffix so traceability is preserved. | +| `sources[].title` | FEAT-021: human-readable citation label ("filename, p.N") matching the `[n]` markers in the answer. Set only when the namespace has `citations_enabled`; `null` otherwise. | | `conversation_id` | Echoed back if provided in request | | `context_only` | `true` if LLM failed and raw sources returned | | `no_relevant_context` | `true` if no chunks exceeded relevance threshold | @@ -522,6 +524,7 @@ Response (HTTP 200, JSON): |-------|-------------| | `show_sources` | Server-resolved citation-visibility hint for the widget (FEAT-014). The full `sources` list is always returned regardless; the widget uses the flag to decide whether to render the citations block. See the resolution chain in [Namespaces PATCH](#patch-apiv1adminnamespacesnamespace_idconfig). | | `sources[].document_name` | Filename of the source document. Soft-deleted documents (REQ-057) keep an `(archived)` suffix so traceability is preserved. The field is `null` when the document join returns no row. | +| `sources[].title` | FEAT-021: citation label ("filename, p.N") for the `[n]` markers, `null` unless the namespace has `citations_enabled`. | #### Streaming From 7cce65f0c0d8c0b505b1c354980211b1d75188c7 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Sun, 12 Jul 2026 22:32:44 +0000 Subject: [PATCH 22/31] fix(core): pass SourceRef.title through the query response mapping (FEAT-021) The manual SourceRef -> SourceRefBody mapping in the sync /api/v1/query path dropped the new title field (schema had it, mapping did not), so citations-enabled namespaces got [n] markers in the answer but null titles in sources. Found by the live smoke test; regression asserted in test_query_json_returns_response. Co-Authored-By: Claude Fable 5 --- vektra-core/src/vektra_core/api.py | 1 + vektra-core/tests/test_api.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/vektra-core/src/vektra_core/api.py b/vektra-core/src/vektra_core/api.py index 960831ea..cd6fad6a 100644 --- a/vektra-core/src/vektra_core/api.py +++ b/vektra-core/src/vektra_core/api.py @@ -363,6 +363,7 @@ async def query( citation_id=s.citation_id, document_version=s.document_version, document_name=s.document_name, + title=s.title, # FEAT-021 ) for s in response.sources ], diff --git a/vektra-core/tests/test_api.py b/vektra-core/tests/test_api.py index 59d49613..56c15b38 100644 --- a/vektra-core/tests/test_api.py +++ b/vektra-core/tests/test_api.py @@ -60,6 +60,7 @@ async def _execute(query_req): score=0.9, snippet="Some relevant snippet.", citation_id=uuid4(), + title="lecture-07.pdf, p.3", # FEAT-021 ) ], conversation_id=query_req.conversation_id, @@ -132,6 +133,8 @@ async def test_query_json_returns_response(): assert isinstance(body["sources"], list) assert len(body["sources"]) == 1 assert "citation_id" in body["sources"][0] + # FEAT-021: the title must survive the SourceRef -> SourceRefBody mapping + assert body["sources"][0]["title"] == "lecture-07.pdf, p.3" async def test_query_requires_auth(): From 103db2001d0e6bd57a92d97304523fdc132531e9 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 00:51:25 +0000 Subject: [PATCH 23/31] feat(core): rescue borderline chunks when the relevance filter empties the set (TECH-007) Multi-part and comparative questions get uniformly low cross-encoder scores (each chunk answers only one part of the question), so VEKTRA_MIN_RELEVANCE_SCORE=0.15 wiped the whole candidate set and the pipeline refused despite 90% raw retrieval hit on the category (9/10 multi-chunk eval questions ended with before=5 after=0). Add an opt-in rescue to the retrieval filter (ARCH-056): when the threshold empties the set and VEKTRA_RETRIEVAL_RESCUE_TOP_K > 0, keep the top-N chunks scoring at least VEKTRA_RETRIEVAL_RESCUE_FLOOR and let the LLM arbitrate via strict grounding. Default off: 0-value top_k preserves the current behavior exactly. The retrieval_filter trace step now records the rescued count in all pipelines. Score distributions measured on eval-full show no static threshold separates multi-chunk from adversarial questions (wiped multi-chunk max rerank 0.005-0.088 vs wiped adversarial 0.000-0.142), so the filter cannot discriminate: the LLM already arbitrates for the 4/9 adversarial questions the reranker scores above the threshold today. Refs: TECH-007, ARCH-056, ADR-0021 Co-Authored-By: Claude Fable 5 --- .env.example | 2 + CHANGELOG.md | 1 + docs/reference/configuration.md | 2 + .../src/vektra_core/advanced_pipeline.py | 5 +- vektra-core/src/vektra_core/pipeline.py | 37 ++++++-- vektra-core/tests/test_advanced_pipeline.py | 29 ++++++ vektra-core/tests/test_pipeline.py | 89 ++++++++++++++++++- vektra-shared/src/vektra_shared/config.py | 22 +++++ vektra-shared/tests/test_config.py | 21 +++++ 9 files changed, 197 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 90a215fa..e576fa2d 100644 --- a/.env.example +++ b/.env.example @@ -84,6 +84,8 @@ # VEKTRA_QUERY_PIPELINE=advanced # VEKTRA_MIN_RELEVANCE_SCORE=0.15 # VEKTRA_CHUNK_DEDUP_ENABLED=true +# VEKTRA_RETRIEVAL_RESCUE_TOP_K=0 +# VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.02 # VEKTRA_PARENT_EXPANSION_ENABLED=false # VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index cd842679..fb2613fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Convention (Keep a Changelog 1.1.0): ### Added +- **rag**: optional retrieval-filter rescue for multi-part questions (TECH-007, `VEKTRA_RETRIEVAL_RESCUE_TOP_K` + `VEKTRA_RETRIEVAL_RESCUE_FLOOR`, default off). When `VEKTRA_MIN_RELEVANCE_SCORE` empties the candidate set, keep the top-N chunks above an absolute floor instead of refusing: the cross-encoder scores each partial-answer chunk of a comparative/multi-part question below the threshold (it answers only one part), so on the eval corpus 9/10 multi-chunk questions died at the filter with `before=5 after=0` despite 90% raw retrieval hit. With the rescue, borderline sets reach the LLM, which arbitrates via strict grounding. The `retrieval_filter` trace step now records a `rescued` count. - **rag**: optional per-namespace inline source citations (FEAT-021, `citations_enabled` in the namespace config JSONB via `PATCH /api/v1/admin/namespaces/{id}/config`, default off, advanced pipeline only). When enabled, the system prompt instructs the LLM to add inline `[n]` markers matching the `` elements, the context template carries a `title` attribute ("filename, p.N"), and each returned source includes a `title` field; the learn widget renders the markers as superscripts with a tooltip. Default-off renders byte-identical prompts; `prompt_version` changes anyway because the template files changed (trace comparability note). - **rag**: optional parent chunk expansion in the advanced query pipeline (FEAT-017, `VEKTRA_PARENT_EXPANSION_ENABLED`, default off). With `VEKTRA_CHUNKING_STRATEGY=dual`, retrieved child chunks are replaced with their parent chunk's text after the retrieval filter and before token budgeting; children of the same parent collapse into the highest-scored one. Parent-child linkage is now actually persisted (deterministic `uuid5(doc_id, position)` ids, `parent_id` in the Qdrant payload and in the pgvector column), a new `VectorStoreProvider.retrieve()` fetches chunks by id, and the trace records `children_expanded`/`siblings_merged`/`parents_fetched` in a `parent_expansion` step. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 48313c8f..1a50360d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -81,6 +81,8 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | `VEKTRA_QUERY_PIPELINE` | str | `advanced` | Pipeline implementation: `simple`, `advanced` | | `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.15` | Minimum relevance score for chunk inclusion (0.0-1.0). Safety net filter; top-k is the primary control. | | `VEKTRA_CHUNK_DEDUP_ENABLED` | bool | `true` | Deduplicate overlapping adjacent chunks from the same document | +| `VEKTRA_RETRIEVAL_RESCUE_TOP_K` | int | `0` | When the `VEKTRA_MIN_RELEVANCE_SCORE` filter empties the candidate set, keep this many top-scored chunks above the rescue floor instead of refusing. Multi-part and comparative questions get uniformly low reranker scores (each chunk answers only one part), so with the rescue the LLM arbitrates via grounding instead of the query dying at the filter. `0` disables the rescue. Recommended starting point when enabling: `3`. | +| `VEKTRA_RETRIEVAL_RESCUE_FLOOR` | float | `0.02` | Absolute minimum score for rescued chunks (0.0-1.0): candidates below this are never rescued. Only used when `VEKTRA_RETRIEVAL_RESCUE_TOP_K` > 0. Lower values rescue more multi-part questions but feed more irrelevant context to adversarial ones. | | `VEKTRA_PARENT_EXPANSION_ENABLED` | bool | `false` | Replace retrieved child chunks with their parent chunk text before prompt construction (advanced pipeline only). Requires documents ingested with `VEKTRA_CHUNKING_STRATEGY=dual`. | | `VEKTRA_RESPONSE_TOKEN_RESERVE` | int | `2048` | Tokens reserved for LLM response generation | | `VEKTRA_CONTEXT_CHUNK_RATIO` | float | `0.6` | Fraction of context window allocated to retrieved chunks (0.0-1.0) | diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 82aec7d4..4b969f56 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -349,10 +349,12 @@ async def _run_pre_llm_steps( # Step 5: Retrieval filter (ARCH-056) t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -363,6 +365,7 @@ async def _run_pre_llm_steps( "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index cd950372..1a97aef4 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -143,20 +143,37 @@ def _apply_retrieval_filter( results: list[SearchResult], min_score: float, dedup_enabled: bool, -) -> list[SearchResult]: + rescue_top_k: int = 0, + rescue_floor: float = 0.0, +) -> tuple[list[SearchResult], int]: """Filter by relevance score and remove near-duplicate chunks (ARCH-056). Step 1: Remove chunks below min_score. - Step 2: If dedup_enabled, remove chunks with >80% token overlap with any + Step 2: If the threshold empties the set and rescue_top_k > 0, keep the + top rescue_top_k chunks scoring >= rescue_floor instead (TECH-007). + The threshold is a safety net, not a hard gate: multi-part + questions get uniformly low reranker scores (each chunk answers + only one part), so a borderline set reaches the LLM, which + arbitrates via grounding. + Step 3: If dedup_enabled, remove chunks with >80% token overlap with any already-selected chunk (keeping higher-scoring ones by processing in score-descending order). - Returns results in score-descending order. + Returns (results in score-descending order, count of rescued chunks). """ filtered = [r for r in results if r.score >= min_score] + rescued = 0 + if not filtered and rescue_top_k > 0: + filtered = sorted( + (r for r in results if r.score >= rescue_floor), + key=lambda r: r.score, + reverse=True, + )[:rescue_top_k] + rescued = len(filtered) + if not dedup_enabled: - return filtered + return filtered, rescued # Process by score descending; keep first occurrence of near-duplicates kept: list[SearchResult] = [] @@ -168,7 +185,7 @@ def _apply_retrieval_filter( if not is_dup: kept.append(candidate) - return kept + return kept, rescued # --------------------------------------------------------------------------- @@ -417,10 +434,12 @@ async def execute( # Step 3: Retrieval filter t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -431,6 +450,7 @@ async def execute( "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) @@ -733,10 +753,12 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] # Step 3: Retrieval filter t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -747,6 +769,7 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index 249da06b..83d4cd50 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -155,6 +155,35 @@ async def test_execute_no_relevant_context(): assert response.answer is None +async def test_execute_rescue_keeps_borderline_chunks(): + """With rescue enabled, a set wiped by the threshold reaches the LLM (TECH-007).""" + results = [ + _make_search_result(0.09, "partial answer one"), + _make_search_result(0.07, "partial answer two"), + _make_search_result(0.01, "below the rescue floor"), + ] + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config( + VEKTRA_RETRIEVAL_RESCUE_TOP_K=3, + VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.02, + ), + ) + response, trace = await pipeline.execute( + QueryRequest(question="multi-part question") + ) + + assert response.no_relevant_context is False + assert response.answer == "The answer." + assert len(response.sources) == 2 + filter_step = next(s for s in trace.steps if s.name == "retrieval_filter") + assert filter_step.metadata["rescued"] == 2 + assert filter_step.metadata["after"] == 2 + + async def test_execute_no_relevant_context_hybrid_calls_llm(): """In hybrid mode, LLM is called even when no chunks pass threshold.""" results = [_make_search_result(0.1, "irrelevant")] diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index ba03600c..db3a6e05 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -135,10 +135,13 @@ def test_retrieval_filter_removes_low_score(): _make_search_result(0.2, "bad chunk"), _make_search_result(0.8, "great chunk"), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) + filtered, rescued = _apply_retrieval_filter( + results, min_score=0.3, dedup_enabled=False + ) scores = [r.score for r in filtered] assert all(s >= 0.3 for s in scores) assert len(filtered) == 2 + assert rescued == 0 def test_retrieval_filter_dedup_removes_near_duplicate(): @@ -148,7 +151,7 @@ def test_retrieval_filter_dedup_removes_near_duplicate(): _make_search_result(0.9, text_a), _make_search_result(0.7, text_b), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=True) + filtered, _ = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=True) assert len(filtered) == 1 assert filtered[0].score == 0.9 # higher-scoring one kept @@ -160,10 +163,90 @@ def test_retrieval_filter_no_dedup_keeps_all(): _make_search_result(0.9, text_a), _make_search_result(0.7, text_b), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) + filtered, _ = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) assert len(filtered) == 2 +def test_retrieval_filter_rescue_disabled_by_default(): + results = [ + _make_search_result(0.09, "part one"), + _make_search_result(0.07, "part two"), + ] + filtered, rescued = _apply_retrieval_filter( + results, min_score=0.15, dedup_enabled=False + ) + assert filtered == [] + assert rescued == 0 + + +def test_retrieval_filter_rescue_keeps_top_k_above_floor(): + results = [ + _make_search_result(0.004, "below floor"), + _make_search_result(0.09, "part one"), + _make_search_result(0.05, "part three"), + _make_search_result(0.07, "part two"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert [r.score for r in filtered] == [0.09, 0.07, 0.05] + assert rescued == 3 + + +def test_retrieval_filter_rescue_all_below_floor_returns_empty(): + results = [ + _make_search_result(0.01, "noise"), + _make_search_result(0.005, "more noise"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert filtered == [] + assert rescued == 0 + + +def test_retrieval_filter_rescue_not_triggered_when_survivors_exist(): + results = [ + _make_search_result(0.5, "strong chunk"), + _make_search_result(0.09, "weak chunk"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert [r.score for r in filtered] == [0.5] + assert rescued == 0 + + +def test_retrieval_filter_rescued_chunks_pass_through_dedup(): + text = "the quick brown fox jumps over the lazy dog and more words here" + results = [ + _make_search_result(0.09, text), + _make_search_result(0.07, text + " again"), # >80% overlap with the first + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=True, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert len(filtered) == 1 + assert filtered[0].score == 0.09 + assert rescued == 2 # rescued before dedup + + # --------------------------------------------------------------------------- # Pipeline execute() tests # --------------------------------------------------------------------------- diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 987b5903..9d5b00ad 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -197,6 +197,19 @@ class QueryPipelineConfig(BaseSettings): alias="VEKTRA_CHUNK_DEDUP_ENABLED", description="Enable overlap deduplication for adjacent chunks from the same document.", ) + retrieval_rescue_top_k: int = Field( + 0, + ge=0, + alias="VEKTRA_RETRIEVAL_RESCUE_TOP_K", + description="When min_relevance_score empties the candidate set, keep this many top-scored chunks above the rescue floor instead of refusing (TECH-007). 0 disables rescue (default).", + ) + retrieval_rescue_floor: float = Field( + 0.02, + ge=0.0, + le=1.0, + alias="VEKTRA_RETRIEVAL_RESCUE_FLOOR", + description="Absolute minimum score for rescued chunks: candidates below this are never rescued. Only used when retrieval_rescue_top_k > 0.", + ) parent_expansion_enabled: bool = Field( False, alias="VEKTRA_PARENT_EXPANSION_ENABLED", @@ -498,6 +511,8 @@ class VektraSettings(BaseSettings): query_pipeline: str = Field("advanced", alias="VEKTRA_QUERY_PIPELINE") min_relevance_score: float = Field(0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE") chunk_dedup_enabled: bool = Field(True, alias="VEKTRA_CHUNK_DEDUP_ENABLED") + retrieval_rescue_top_k: int = Field(0, alias="VEKTRA_RETRIEVAL_RESCUE_TOP_K") + retrieval_rescue_floor: float = Field(0.02, alias="VEKTRA_RETRIEVAL_RESCUE_FLOOR") parent_expansion_enabled: bool = Field( False, alias="VEKTRA_PARENT_EXPANSION_ENABLED" ) @@ -580,6 +595,13 @@ def validate_relevance_score(cls, v: float) -> float: raise ValueError(f"min_relevance_score must be between 0 and 1, got {v}") return v + @field_validator("retrieval_rescue_floor") + @classmethod + def validate_rescue_floor(cls, v: float) -> float: + if not 0.0 <= v <= 1.0: + raise ValueError(f"retrieval_rescue_floor must be between 0 and 1, got {v}") + return v + @field_validator("prompt_grounding_mode") @classmethod def validate_prompt_grounding_mode(cls, v: str) -> str: diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 9a3dd1fd..51c2cf1e 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -77,6 +77,27 @@ def test_eval_mode_from_env(self) -> None: cfg = QueryPipelineConfig(VEKTRA_EVAL_MODE=True) assert cfg.eval_mode is True + def test_rescue_defaults_off(self) -> None: + cfg = QueryPipelineConfig() + assert cfg.retrieval_rescue_top_k == 0 + assert cfg.retrieval_rescue_floor == 0.02 + + def test_rescue_from_env(self) -> None: + cfg = QueryPipelineConfig( + VEKTRA_RETRIEVAL_RESCUE_TOP_K=3, + VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.005, + ) + assert cfg.retrieval_rescue_top_k == 3 + assert cfg.retrieval_rescue_floor == 0.005 + + def test_rescue_top_k_negative_invalid(self) -> None: + with pytest.raises(ValidationError, match="VEKTRA_RETRIEVAL_RESCUE_TOP_K"): + QueryPipelineConfig(VEKTRA_RETRIEVAL_RESCUE_TOP_K=-1) + + def test_rescue_floor_above_one_invalid(self) -> None: + with pytest.raises(ValidationError, match="VEKTRA_RETRIEVAL_RESCUE_FLOOR"): + QueryPipelineConfig(VEKTRA_RETRIEVAL_RESCUE_FLOOR=1.5) + def test_debug_log_queries_from_env(self) -> None: cfg = QueryPipelineConfig(VEKTRA_DEBUG_LOG_QUERIES=True) assert cfg.debug_log_queries is True From 62f5b9b6d53edb255239085981634b24d24e3139 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 01:13:11 +0000 Subject: [PATCH 24/31] docs(backlog): mark TECH-007 completed with measurement evidence Score distributions, mitigation rationale, and A/B numbers recorded in the sprint plan Notes and in vektra-internal (20260713-tech007-retrieval-rescue.md + eval artifacts). Co-Authored-By: Claude Fable 5 --- .s2s/BACKLOG.md | 13 ++++++++----- .s2s/plans/20260712-sprint3-rag-quality.md | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index daf722a1..3bd8e9fb 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -371,20 +371,23 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. ### TECH-007: Multi-part questions wiped by rerank+threshold funnel (multi-chunk collapse root cause) -**Status**: planned | **Priority**: high | **Created**: 2026-07-12 +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-13 | **PR**: #92 **Origin**: FEAT-017 measurement (plan `20260712-sprint3-rag-quality`) - expansion turned out to be downstream of the real failure. +**Evidence**: `vektra-internal/stack/20260713-tech007-retrieval-rescue.md` (+ per-question artifacts in `20260713-tech007-eval-artifacts/`) **Context**: on `eval-full`, 9/10 multi-chunk questions end with `retrieval_filter before=5 after=0` → `no_relevant_context` → refusal, despite 90% raw retrieval hit for the category. Cause: bge-reranker-v2-m3 scores each partial-answer chunk of a comparative/multi-part question low (each chunk answers only one part), and `VEKTRA_MIN_RELEVANCE_SCORE=0.15` — calibrated in the tuning sprint on single-fact questions (DEBT-010) — wipes the entire candidate set. Evidence (MC-01, eval mode traces): max reranker score 0.088 on a candidate whose raw RRF score was 0.61. Parent expansion (FEAT-017) never runs because zero results survive the filter. **Candidate directions** (evaluate, do not assume): (a) floor semantics - keep top-N post-rerank chunks regardless of threshold when the raw retrieval score was strong (e.g. min(top_k, after_rerank) >= 2); (b) per-category or per-score-source thresholds (reranker scores are not calibrated on the same scale as RRF); (c) query decomposition for multi-part questions (rewrite step already exists, ARCH-061); (d) rescore against the parent text instead of the child (combines with FEAT-017). +**Resolution**: measured score distributions showed the collapse was wider than multi-chunk (also 4 reasoning + 2 factual wiped, hence grounded 35/55) and that no static threshold separates multi-chunk from adversarial (wiped MC max-rerank 0.005-0.088 vs wiped ADV 0.000-0.142, full overlap) — the filter cannot discriminate, and the reranker already passes 4-5/9 adversarial to the LLM today. Chose direction (a) in minimal form: **rescue only-when-empty** (`VEKTRA_RETRIEVAL_RESCUE_TOP_K`, default 0 = off; `VEKTRA_RETRIEVAL_RESCUE_FLOOR`, default 0.02): when the threshold empties the set, keep the top-N chunks above the floor and let strict grounding arbitrate. Discarded: (b) does not discriminate; (c) larger feature, downstream; (d) per-query cost, combinable later. Measured with `top_k=3, floor=0.005`: grounded 35/55 → 54/55, factual 19→21/21, reasoning 11→15/15, multi-chunk 1→10/10 by the harness metric — honestly: 2-3/10 substantially complete answers, 7 informed refusals that explain the gap (candidates for bi-document comparatives never include chunks of both documents: a candidate-coverage limit upstream of the filter, not a funnel issue). Adversarial: 0 answered-without-context, 0 hallucinations on manual review of all 9 (rescued ones give informed refusals or correct corrective answers). `top_k=5` control run equivalent within LLM variance. Latency unchanged. + **Traceability**: ARCH-056 (retrieval quality controls), ADR-0021, DEBT-010, FEAT-017, TECH-005 **Acceptance criteria**: -- [ ] Reproduce with the eval harness and document the score distributions per category -- [ ] Chosen mitigation implemented behind config, default preserving current single-fact behavior -- [ ] `eval-full` multi-chunk grounded moves from 0-1/10 without regressing factual (19/21) or adversarial refusals (no answered-without-context) -- [ ] Decision and numbers recorded in the sprint plan and vektra-internal +- [x] Reproduce with the eval harness and document the score distributions per category +- [x] Chosen mitigation implemented behind config, default preserving current single-fact behavior +- [x] `eval-full` multi-chunk grounded moves from 0-1/10 without regressing factual (19/21) or adversarial refusals (no answered-without-context) +- [x] Decision and numbers recorded in the sprint plan and vektra-internal --- diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index 5b6e7d4b..86ae3f93 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -301,3 +301,25 @@ the bundle and a manual smoke in the Moodle dev stack. "diversi da quelli già citati" rewrite) but degrades gracefully via history. Chunk exclusion today would raise refusals on follow-ups; revisit after TECH-007. Full per-turn evidence in the FEAT-018 backlog entry. +- 2026-07-13: **TECH-007 resolved (post-sprint, PR #92)** — retrieval-filter + rescue, `feat/tech-007-retrieval-rescue`. Reproduction with eval-mode traces + on all 55 questions showed (1) the collapse was wider than multi-chunk: + 4 reasoning + 2 factual also wiped (that is why grounded was 35/55); + (2) no static threshold separates multi-chunk from adversarial (wiped MC + max-rerank 0.005-0.088 vs wiped ADV 0.000-0.142, full overlap) — the filter + cannot discriminate, and the reranker already passes 4-5/9 adversarial to + the LLM (0.36-0.84) whose strict grounding answers correctively. Mitigation: + rescue only-when-empty behind `VEKTRA_RETRIEVAL_RESCUE_TOP_K` (default 0 = + off) + `VEKTRA_RETRIEVAL_RESCUE_FLOOR` (default 0.02); `retrieval_filter` + trace step gains `rescued`. Measured with top_k=3/floor=0.005 (same corpus, + expansion on): grounded 35/55 -> **54/55**, factual 19 -> 21/21 (kw 45/50), + reasoning 11 -> 15/15 (kw 41/46), multi-chunk 1 -> 10/10 by harness metric + (honest reading: 2-3/10 substantially complete, 7 informed refusals that + correctly explain the missing half — bi-document comparatives never get + chunks of both documents among the 20 candidates: candidate-coverage limit, + upstream of the filter). Adversarial: 0 answered-without-context, 0 + hallucinations (manual review 9/9). top_k=5 control run equivalent within + LLM variance; p50 unchanged (~3.9s). Full report: + `vektra-internal/stack/20260713-tech007-retrieval-rescue.md`; per-question + artifacts in `20260713-tech007-eval-artifacts/`. Dev `.env` adds + `VEKTRA_RETRIEVAL_RESCUE_TOP_K=3`, `VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.005`. From 68e7481e7bcb1c8d68df4cbeb355759acc3d770f Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 01:21:42 +0000 Subject: [PATCH 25/31] fix(shared): reject negative retrieval_rescue_top_k in the settings mirror Review: gemini #3567544954, coderabbit #3567551399. QueryPipelineConfig already enforces ge=0; the VektraSettings mirror (startup validation, ARCH-057) now rejects it too, symmetric with the rescue floor validator. Co-Authored-By: Claude Fable 5 --- vektra-shared/src/vektra_shared/config.py | 7 +++++++ vektra-shared/tests/test_config.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 9d5b00ad..d7cc1c4a 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -602,6 +602,13 @@ def validate_rescue_floor(cls, v: float) -> float: raise ValueError(f"retrieval_rescue_floor must be between 0 and 1, got {v}") return v + @field_validator("retrieval_rescue_top_k") + @classmethod + def validate_rescue_top_k(cls, v: int) -> int: + if v < 0: + raise ValueError(f"retrieval_rescue_top_k must be >= 0, got {v}") + return v + @field_validator("prompt_grounding_mode") @classmethod def validate_prompt_grounding_mode(cls, v: str) -> str: diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 51c2cf1e..e307bdc5 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -223,6 +223,14 @@ def test_context_chunk_ratio_one_invalid(self) -> None: with pytest.raises(ValidationError, match="context_chunk_ratio"): self._make(VEKTRA_CONTEXT_CHUNK_RATIO=1.0) + def test_rescue_top_k_negative_invalid(self) -> None: + with pytest.raises(ValidationError, match="retrieval_rescue_top_k"): + self._make(VEKTRA_RETRIEVAL_RESCUE_TOP_K=-1) + + def test_rescue_floor_above_one_invalid(self) -> None: + with pytest.raises(ValidationError, match="retrieval_rescue_floor"): + self._make(VEKTRA_RETRIEVAL_RESCUE_FLOOR=1.5) + def test_min_relevance_score_valid_zero(self) -> None: s = self._make(VEKTRA_MIN_RELEVANCE_SCORE=0.0) assert s.min_relevance_score == 0.0 From 380a660cbeaf0092c999c6762959552f25546ab8 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 01:21:42 +0000 Subject: [PATCH 26/31] style(core): document the actual ordering guarantee of the retrieval filter Review: coderabbit nitpick on the docstring promising score-descending order that the plain-threshold path does not itself enforce. Co-Authored-By: Claude Fable 5 --- vektra-core/src/vektra_core/pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index 1a97aef4..2e3ea76c 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -159,7 +159,10 @@ def _apply_retrieval_filter( already-selected chunk (keeping higher-scoring ones by processing in score-descending order). - Returns (results in score-descending order, count of rescued chunks). + Returns (filtered results, count of rescued chunks). The rescue and dedup + paths return results in score-descending order; the plain-threshold path + preserves the caller's ordering (already score-descending after search or + rerank). """ filtered = [r for r in results if r.score >= min_score] From c07e4e22a3c44cbac452aa6db11c05fd951ba363 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 01:37:34 +0000 Subject: [PATCH 27/31] feat(rag): remote TEI embedding and reranker providers (FEAT-024) VEKTRA_EMBEDDING_PROVIDER=tei was documented but no provider existed: main.py unconditionally instantiated in-process SentenceTransformers, so every instance duplicates embedding compute and long-window models served by shared TEI instances (bge-m3, 8192-token window vs MiniLM's 128) were unreachable. - TEIEmbeddingProvider (vektra-index): native /embed API, batch chunking to the TEI client batch limit, dimensions from /info with an /embed probe fallback, Bearer auth, health check. - TEIRerankerService (vektra-core): TEI /rerank with the same RerankResult semantics as the in-process service (BUG-015 score propagation, sigmoid normalization when raw scores are detected); shared _build_rerank_result helper and RerankerProtocol for typing. - Dimension plumbing: the Qdrant collection is sized from the active embedding provider instead of the hardcoded 384 default, and ensure_collection fails fast with a clear message on a dimension mismatch with an existing collection. - The cohere rerank option now passes VEKTRA_RERANK_API_KEY through; it never received a key before (dead as wired). Refs: FEAT-024, ADR-0013, ARCH-035, ARCH-036 Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + uv.lock | 4 + vektra-app/src/vektra_app/main.py | 37 ++++- vektra-core/pyproject.toml | 1 + .../src/vektra_core/advanced_pipeline.py | 4 +- vektra-core/src/vektra_core/reranker.py | 137 ++++++++++++++---- vektra-core/tests/test_reranker.py | 77 +++++++++- vektra-index/pyproject.toml | 1 + .../src/vektra_index/providers/qdrant.py | 18 ++- .../src/vektra_index/providers/tei.py | 121 ++++++++++++++++ vektra-index/tests/test_tei_provider.py | 137 ++++++++++++++++++ vektra-shared/src/vektra_shared/config.py | 29 +++- 12 files changed, 525 insertions(+), 42 deletions(-) create mode 100644 vektra-index/src/vektra_index/providers/tei.py create mode 100644 vektra-index/tests/test_tei_provider.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2613fc..6e719f54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Convention (Keep a Changelog 1.1.0): ### Added +- **rag**: remote embedding and reranking via HuggingFace Text Embeddings Inference (FEAT-024). `VEKTRA_EMBEDDING_PROVIDER=tei` embeds through a TEI instance (`VEKTRA_TEI_URL`/`VEKTRA_TEI_API_KEY`, native `/embed` API) instead of in-process sentence-transformers, enabling shared host inference and long-window models (bge-m3: 8192 tokens vs MiniLM's 128, which silently truncates 500-token chunks today). `VEKTRA_RERANK_PROVIDER=tei` reranks through TEI `/rerank` (`VEKTRA_RERANK_TEI_URL`/`VEKTRA_RERANK_TEI_API_KEY`). The Qdrant collection is now sized from the active embedding provider's dimensions instead of a hardcoded 384 (latent bug for any non-384 model), with a clear startup error on dimension mismatch against an existing collection. The `cohere` rerank option now actually passes its API key (`VEKTRA_RERANK_API_KEY`); it was dead as wired. - **rag**: optional retrieval-filter rescue for multi-part questions (TECH-007, `VEKTRA_RETRIEVAL_RESCUE_TOP_K` + `VEKTRA_RETRIEVAL_RESCUE_FLOOR`, default off). When `VEKTRA_MIN_RELEVANCE_SCORE` empties the candidate set, keep the top-N chunks above an absolute floor instead of refusing: the cross-encoder scores each partial-answer chunk of a comparative/multi-part question below the threshold (it answers only one part), so on the eval corpus 9/10 multi-chunk questions died at the filter with `before=5 after=0` despite 90% raw retrieval hit. With the rescue, borderline sets reach the LLM, which arbitrates via strict grounding. The `retrieval_filter` trace step now records a `rescued` count. - **rag**: optional per-namespace inline source citations (FEAT-021, `citations_enabled` in the namespace config JSONB via `PATCH /api/v1/admin/namespaces/{id}/config`, default off, advanced pipeline only). When enabled, the system prompt instructs the LLM to add inline `[n]` markers matching the `` elements, the context template carries a `title` attribute ("filename, p.N"), and each returned source includes a `title` field; the learn widget renders the markers as superscripts with a tooltip. Default-off renders byte-identical prompts; `prompt_version` changes anyway because the template files changed (trace comparability note). - **rag**: optional parent chunk expansion in the advanced query pipeline (FEAT-017, `VEKTRA_PARENT_EXPANSION_ENABLED`, default off). With `VEKTRA_CHUNKING_STRATEGY=dual`, retrieved child chunks are replaced with their parent chunk's text after the retrieval filter and before token budgeting; children of the same parent collapse into the highest-scored one. Parent-child linkage is now actually persisted (deterministic `uuid5(doc_id, position)` ids, `parent_id` in the Qdrant payload and in the pgvector column), a new `VectorStoreProvider.retrieve()` fetches chunks by id, and the trace records `children_expanded`/`siblings_merged`/`parents_fetched` in a `parent_expansion` step. diff --git a/uv.lock b/uv.lock index d0e7deed..112aae6f 100644 --- a/uv.lock +++ b/uv.lock @@ -5236,6 +5236,7 @@ version = "0.5.1" source = { editable = "vektra-core" } dependencies = [ { name = "fastapi" }, + { name = "httpx" }, { name = "jinja2" }, { name = "litellm" }, { name = "presidio-analyzer" }, @@ -5256,6 +5257,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "fastapi", specifier = ">=0.115" }, + { name = "httpx", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "litellm", specifier = ">=1.83.10" }, { name = "presidio-analyzer", specifier = ">=2.2" }, @@ -5280,6 +5282,7 @@ source = { editable = "vektra-index" } dependencies = [ { name = "asyncpg" }, { name = "fastapi" }, + { name = "httpx" }, { name = "pgvector" }, { name = "pydantic" }, { name = "sentence-transformers" }, @@ -5310,6 +5313,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.29" }, { name = "fastapi", specifier = ">=0.115" }, { name = "fastembed", marker = "extra == 'sparse'", specifier = ">=0.4" }, + { name = "httpx", specifier = ">=0.27" }, { name = "pgvector", specifier = ">=0.3" }, { name = "pydantic", specifier = ">=2,<3" }, { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = ">=1.12" }, diff --git a/vektra-app/src/vektra_app/main.py b/vektra-app/src/vektra_app/main.py index 25a825fa..f112f9a8 100644 --- a/vektra-app/src/vektra_app/main.py +++ b/vektra-app/src/vektra_app/main.py @@ -31,6 +31,7 @@ from vektra_shared.config import QueryPipelineConfig, VektraSettings from vektra_shared.db import init_db from vektra_shared.errors import ERR_CONFIG_001, ErrorCategory, ErrorResponse +from vektra_shared.protocols import EmbeddingProvider from vektra_shared.registry import ProviderRegistry from vektra_shared.startup import ( StartupValidationError, @@ -126,15 +127,31 @@ async def _step_5_register_providers( registry.register("llm", "default", llm_provider) # --- Embedding --- - from vektra_index.providers.sentence_transformers import ( - SentenceTransformersProvider, - ) + embedding_provider: EmbeddingProvider + if settings.embedding_provider == "tei": + from vektra_index.providers.tei import TEIEmbeddingProvider - embedding_provider = SentenceTransformersProvider( - model_name=settings.embedding_model - ) - registry.register("embedding", "default", embedding_provider) - registry.register("embedding", "sentence-transformers", embedding_provider) + embedding_provider = TEIEmbeddingProvider( + url=settings.tei_url, + api_key=settings.tei_api_key, + ) + registry.register("embedding", "default", embedding_provider) + registry.register("embedding", "tei", embedding_provider) + log.info( + "embedding_registered", + provider="tei", + url=_redact_url(settings.tei_url), + ) + else: + from vektra_index.providers.sentence_transformers import ( + SentenceTransformersProvider, + ) + + embedding_provider = SentenceTransformersProvider( + model_name=settings.embedding_model + ) + registry.register("embedding", "default", embedding_provider) + registry.register("embedding", "sentence-transformers", embedding_provider) # --- Vector store --- from vektra_index.adapters import VectorStoreServiceAdapter @@ -168,6 +185,10 @@ async def _step_5_register_providers( api_key=settings.qdrant_api_key, collection_name=settings.qdrant_collection, active_index_version=settings.active_index_version, + # FEAT-024: size the collection from the active embedding model + # instead of the hardcoded 384 default (latent bug for any + # non-384 model; bge-m3 via TEI is 1024). + dense_dimensions=embedding_provider.dimensions(), ) registry.register("vector_store", "default", qdrant_provider) registry.register("vector_store", "qdrant", qdrant_provider) diff --git a/vektra-core/pyproject.toml b/vektra-core/pyproject.toml index f1dabe5b..b8259147 100644 --- a/vektra-core/pyproject.toml +++ b/vektra-core/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "presidio-analyzer>=2.2", "presidio-anonymizer>=2.2", "rerankers[flashrank]>=0.5", + "httpx>=0.27", ] [build-system] diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 4b969f56..14dfec3d 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -39,7 +39,7 @@ _history_to_messages, _trace_to_dict, ) -from vektra_core.reranker import RerankerService +from vektra_core.reranker import RerankerProtocol from vektra_core.templates import TemplateRenderer from vektra_shared.config import LLMConfig, QueryPipelineConfig from vektra_shared.protocols import ( @@ -87,7 +87,7 @@ def __init__( renderer: TemplateRenderer, pipeline_config: QueryPipelineConfig, sparse_embedding: SparseEmbeddingProvider | None = None, - reranker: RerankerService | None = None, + reranker: RerankerProtocol | None = None, ) -> None: self._embedding = embedding self._vector_store = vector_store diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 87db0be4..a8cde2bf 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -14,7 +14,9 @@ import asyncio import dataclasses import math +from typing import Protocol, runtime_checkable +import httpx import structlog from vektra_shared.config import RerankConfig @@ -49,6 +51,54 @@ class RerankResult: ] # (chunk_id, reranker_score, original_score) +@runtime_checkable +class RerankerProtocol(Protocol): + """Common interface of the in-process and remote reranker services.""" + + async def rerank( + self, + query: str, + results: list[SearchResult], + top_k: int, + ) -> RerankResult: ... + + +def _build_rerank_result( + results: list[SearchResult], + ordered: list[tuple[int, float]], + top_k: int, +) -> RerankResult: + """Build a RerankResult from (candidate_index, raw_score) pairs. + + Detects whether normalization is needed: FlashRank and TEI (with + raw_scores=false) produce sigmoid scores in [0, 1]; cross-encoder + logits can be negative or > 1. + """ + all_raw = [score for _, score in ordered] + needs_sigmoid = any(s < 0.0 or s > 1.0 for s in all_raw) + + all_scores: list[tuple[str, float, float]] = [] + reranked: list[SearchResult] = [] + + for idx, raw in ordered: + original = results[idx] + normalized = _sigmoid(raw) if needs_sigmoid else raw + all_scores.append( + (original.chunk_id, round(normalized, 4), round(original.score, 4)) + ) + + if len(reranked) < top_k: + reranked.append( + dataclasses.replace( + original, + score=normalized, + original_score=original.score, + ) + ) + + return RerankResult(top_k=reranked, all_scores=all_scores) + + class RerankerService: """Wraps the rerankers library for scoring and reordering search results.""" @@ -77,49 +127,78 @@ async def rerank( docs=docs, ) - # Score ALL candidates and detect whether normalization is needed. - # FlashRank produces sigmoid scores in [0, 1]; cross-encoder - # produces raw logits that can be negative or > 1. - all_items = ranked.results - all_raw = [float(item.score) for item in all_items] - needs_sigmoid = any(s < 0.0 or s > 1.0 for s in all_raw) - - all_scores: list[tuple[str, float, float]] = [] - reranked: list[SearchResult] = [] - - for item in all_items: - original = results[item.doc_id] - raw = float(item.score) - normalized = _sigmoid(raw) if needs_sigmoid else raw - all_scores.append( - (original.chunk_id, round(normalized, 4), round(original.score, 4)) - ) + ordered = [(item.doc_id, float(item.score)) for item in ranked.results] + return _build_rerank_result(results, ordered, top_k) - if len(reranked) < top_k: - reranked.append( - dataclasses.replace( - original, - score=normalized, - original_score=original.score, - ) - ) - return RerankResult(top_k=reranked, all_scores=all_scores) +class TEIRerankerService: + """Reranker backed by a TEI /rerank endpoint (FEAT-024). + + One TEI instance serves one reranker model (e.g. bge-reranker-v2-m3). + POST /rerank {"query", "texts", "raw_scores": false} returns + [{"index", "score"}] sorted by score descending, scores in [0, 1]. + """ + def __init__( + self, + *, + url: str, + api_key: str | None = None, + timeout_s: float = 30.0, + _client: httpx.AsyncClient | None = None, + ) -> None: + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + self._client = _client or httpx.AsyncClient( + base_url=url.rstrip("/"), headers=headers, timeout=timeout_s + ) + + async def rerank( + self, + query: str, + results: list[SearchResult], + top_k: int, + ) -> RerankResult: + """Rerank search results via the remote TEI cross-encoder.""" + if not results: + return RerankResult(top_k=[], all_scores=[]) + + resp = await self._client.post( + "/rerank", + json={ + "query": query, + "texts": [r.text_snippet for r in results], + "raw_scores": False, + }, + ) + resp.raise_for_status() + ranked = resp.json() + + ordered = [(int(item["index"]), float(item["score"])) for item in ranked] + return _build_rerank_result(results, ordered, top_k) -def create_reranker(config: RerankConfig) -> RerankerService | None: - """Create a RerankerService from config. Returns None if unavailable.""" + +def create_reranker(config: RerankConfig) -> RerankerProtocol | None: + """Create a reranker service from config. Returns None if unavailable.""" if not config.enabled: log.info("reranker_disabled") return None + if config.provider == "tei": + log.info("reranker_loaded", provider="tei", url=config.tei_url) + return TEIRerankerService(url=config.tei_url, api_key=config.tei_api_key) + model_type = _PROVIDER_TO_MODEL_TYPE.get(config.provider, config.provider) model_name = config.model or _default_model_for_provider(config.provider) try: from rerankers import Reranker # type: ignore[import-untyped] - ranker = Reranker(model_name, model_type=model_type, verbose=0) + # API-based providers (cohere) need the key passed through; + # without it the option was dead as wired (FEAT-024). + kwargs: dict[str, str] = {} + if config.api_key: + kwargs["api_key"] = config.api_key + ranker = Reranker(model_name, model_type=model_type, verbose=0, **kwargs) if ranker is None: log.warning( "reranker_init_failed", diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index ca631da4..59ef0db6 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -1,12 +1,17 @@ -"""Unit tests for RerankerService and create_reranker factory.""" +"""Unit tests for RerankerService, TEIRerankerService and create_reranker.""" +import json from types import SimpleNamespace from unittest.mock import MagicMock, patch from uuid import uuid4 +import httpx +import pytest + from vektra_core.reranker import ( RerankerService, RerankResult, + TEIRerankerService, _default_model_for_provider, _sigmoid, create_reranker, @@ -242,3 +247,73 @@ def test_sigmoid_extreme_values_no_overflow(): """Numerically stable sigmoid must not overflow on extreme logits.""" assert _sigmoid(1000.0) == 1.0 assert _sigmoid(-1000.0) == 0.0 + + +# --------------------------------------------------------------------------- +# TEIRerankerService (FEAT-024) +# --------------------------------------------------------------------------- + + +def _tei_client(handler) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + + +async def test_tei_rerank_empty_results(): + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(None)) + result = await service.rerank("query", [], top_k=5) + assert result.top_k == [] + assert result.all_scores == [] + + +async def test_tei_rerank_orders_and_normalizes(): + results = [ + _make_result(0.5, "doc A"), + _make_result(0.3, "doc B"), + _make_result(0.9, "doc C"), + ] + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert request.url.path == "/rerank" + assert body["query"] == "test query" + assert body["texts"] == ["doc A", "doc B", "doc C"] + assert body["raw_scores"] is False + return httpx.Response( + 200, + json=[ + {"index": 2, "score": 0.95}, + {"index": 0, "score": 0.80}, + {"index": 1, "score": 0.10}, + ], + ) + + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(handler)) + result = await service.rerank("test query", results, top_k=2) + + assert len(result.top_k) == 2 + assert result.top_k[0].chunk_id == results[2].chunk_id + assert result.top_k[0].score == 0.95 + assert result.top_k[0].original_score == 0.9 # BUG-015 preserved + assert result.top_k[1].chunk_id == results[0].chunk_id + assert len(result.all_scores) == 3 + + +async def test_tei_rerank_raises_on_http_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(handler)) + with pytest.raises(httpx.HTTPStatusError): + await service.rerank("q", [_make_result(0.5)], top_k=2) + + +def test_create_reranker_tei_provider(): + config = RerankConfig( + VEKTRA_RERANK_ENABLED=True, + VEKTRA_RERANK_PROVIDER="tei", + VEKTRA_RERANK_TEI_URL="http://tei.test", + ) + reranker = create_reranker(config) + assert isinstance(reranker, TEIRerankerService) diff --git a/vektra-index/pyproject.toml b/vektra-index/pyproject.toml index b891dd83..9bcb0cc9 100644 --- a/vektra-index/pyproject.toml +++ b/vektra-index/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "sentence-transformers>=3.0", "torch>=2.2.0", "pgvector>=0.3", + "httpx>=0.27", ] [project.optional-dependencies] diff --git a/vektra-index/src/vektra_index/providers/qdrant.py b/vektra-index/src/vektra_index/providers/qdrant.py index a5028de9..71dbf39e 100644 --- a/vektra-index/src/vektra_index/providers/qdrant.py +++ b/vektra-index/src/vektra_index/providers/qdrant.py @@ -106,13 +106,29 @@ async def ensure_collection(self) -> None: """Create the collection if it doesn't exist. Called during startup validation. Configures named vectors - for dense and sparse search. + for dense and sparse search. If the collection already exists, + verifies its dense vector size matches the active embedding model + (FEAT-024): a silent mismatch would fail on every upsert/search + with an opaque Qdrant error, so fail fast with a clear message. """ from qdrant_client import models collections = await self._client.get_collections() existing = {c.name for c in collections.collections} if self._collection_name in existing: + info = await self._client.get_collection(self._collection_name) + vectors = info.config.params.vectors + dense = vectors.get("dense") if isinstance(vectors, dict) else None + existing_size = getattr(dense, "size", None) + if existing_size is not None and existing_size != self._dense_dimensions: + raise ValueError( + f"Qdrant collection '{self._collection_name}' has dense " + f"vectors of size {existing_size}, but the active embedding " + f"model produces {self._dense_dimensions} dimensions. " + "Changing the embedding model requires re-ingesting into a " + "new collection (set VEKTRA_QDRANT_COLLECTION) or deleting " + "the existing one." + ) return try: diff --git a/vektra-index/src/vektra_index/providers/tei.py b/vektra-index/src/vektra_index/providers/tei.py new file mode 100644 index 00000000..75f2b165 --- /dev/null +++ b/vektra-index/src/vektra_index/providers/tei.py @@ -0,0 +1,121 @@ +"""TEIEmbeddingProvider: EmbeddingProvider over HuggingFace Text Embeddings Inference. + +Remote embedding via a TEI instance (FEAT-024, ADR-0013). Lets deployments +reuse a shared inference service (e.g. bge-m3 on the host) instead of +duplicating in-process CPU embedding per container, and unlocks models +whose sequence window exceeds the in-process default (bge-m3: 8192 tokens +vs 128 for paraphrase-multilingual-MiniLM-L12-v2). + +Endpoints used (TEI native API): +- POST /embed {"inputs": [...]} -> [[...], ...] +- GET /info for the embedding size (with an /embed probe fallback) + +Auth: optional Bearer token (TEI --api-key). +""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +import httpx + +from vektra_shared.types import HealthStatus + +logger = logging.getLogger(__name__) + +# TEI rejects batches larger than its --max-client-batch-size (default 32). +_MAX_BATCH = 32 + + +class TEIEmbeddingProvider: + """EmbeddingProvider backed by a remote TEI instance. + + Symmetric encoding: TEI applies the model's own pooling/normalization; + query and document paths use the same endpoint. + """ + + def __init__( + self, + *, + url: str, + api_key: str | None = None, + timeout_s: float = 30.0, + _client: httpx.AsyncClient | None = None, + ) -> None: + self._url = url.rstrip("/") + self._api_key = api_key + self._timeout_s = timeout_s + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + self._client = _client or httpx.AsyncClient( + base_url=self._url, headers=headers, timeout=timeout_s + ) + self._dimensions: int | None = None + + async def _embed_batch(self, texts: list[str]) -> list[list[float]]: + resp = await self._client.post("/embed", json={"inputs": texts}) + resp.raise_for_status() + data: list[list[float]] = resp.json() + return data + + async def embed_documents(self, texts: list[str]) -> list[list[float]]: + """Embed a batch of document passages, chunked to the TEI batch limit.""" + out: list[list[float]] = [] + for i in range(0, len(texts), _MAX_BATCH): + out.extend(await self._embed_batch(texts[i : i + _MAX_BATCH])) + return out + + async def embed_query(self, query: str) -> list[float]: + """Embed a single query string for retrieval.""" + return (await self._embed_batch([query]))[0] + + def dimensions(self) -> int: + """Return the embedding dimensionality, fetched once from the server. + + The Protocol method is synchronous, so this uses a one-off sync HTTP + call (startup/wiring path, not the query hot path). Tries /info + first; TEI versions that do not expose the size fall back to probing + /embed with a single input. + """ + if self._dimensions is not None: + return self._dimensions + + headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {} + with httpx.Client( + base_url=self._url, headers=headers, timeout=self._timeout_s + ) as client: + self._dimensions = self._fetch_dimensions(client) + + logger.info("TEI embedding provider: %s (%d dims)", self._url, self._dimensions) + return self._dimensions + + @staticmethod + def _fetch_dimensions(client: httpx.Client) -> int: + try: + info: dict[str, Any] = client.get("/info").raise_for_status().json() + for key in ("embedding_size", "hidden_size"): + if isinstance(info.get(key), int): + return int(info[key]) + except httpx.HTTPStatusError: + logger.debug("TEI /info unavailable, probing /embed for dimensions") + probe = ( + client.post("/embed", json={"inputs": ["dim probe"]}) + .raise_for_status() + .json() + ) + return len(probe[0]) + + async def health_check(self) -> HealthStatus: + """Verify the TEI server responds and can produce an embedding.""" + try: + start = time.monotonic() + await self.embed_query("health check") + latency_ms = int((time.monotonic() - start) * 1000) + return HealthStatus(status="healthy", latency_ms=latency_ms) + except Exception as exc: + return HealthStatus(status="unhealthy", message=str(exc)) + + async def aclose(self) -> None: + """Release the underlying HTTP client (tests and shutdown).""" + await self._client.aclose() diff --git a/vektra-index/tests/test_tei_provider.py b/vektra-index/tests/test_tei_provider.py new file mode 100644 index 00000000..a733c48c --- /dev/null +++ b/vektra-index/tests/test_tei_provider.py @@ -0,0 +1,137 @@ +"""Unit tests for TEIEmbeddingProvider (FEAT-024). All HTTP mocked.""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +from vektra_index.providers.tei import _MAX_BATCH, TEIEmbeddingProvider + + +def _embed_handler(dim: int = 4): + """MockTransport handler: /embed returns one vector per input.""" + calls: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/embed": + inputs = json.loads(request.content)["inputs"] + calls.append(len(inputs)) + return httpx.Response(200, json=[[0.1] * dim for _ in inputs]) + return httpx.Response(404) + + return handler, calls + + +def _make_provider(handler) -> TEIEmbeddingProvider: + client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + return TEIEmbeddingProvider(url="http://tei.test", _client=client) + + +async def test_embed_query_returns_single_vector(): + handler, _ = _embed_handler(dim=4) + provider = _make_provider(handler) + vec = await provider.embed_query("hello") + assert vec == [0.1] * 4 + await provider.aclose() + + +async def test_embed_documents_chunks_to_batch_limit(): + handler, calls = _embed_handler(dim=4) + provider = _make_provider(handler) + texts = [f"doc {i}" for i in range(_MAX_BATCH + 5)] + vectors = await provider.embed_documents(texts) + assert len(vectors) == _MAX_BATCH + 5 + assert calls == [_MAX_BATCH, 5] + await provider.aclose() + + +async def test_health_check_healthy(): + handler, _ = _embed_handler() + provider = _make_provider(handler) + status = await provider.health_check() + assert status.status == "healthy" + await provider.aclose() + + +async def test_health_check_unhealthy_on_http_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + provider = _make_provider(handler) + status = await provider.health_check() + assert status.status == "unhealthy" + await provider.aclose() + + +def test_fetch_dimensions_from_info(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(200, json={"model_id": "m", "embedding_size": 1024}) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 1024 + + +def test_fetch_dimensions_probe_fallback_when_info_lacks_size(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(200, json={"model_id": "m"}) + if request.url.path == "/embed": + return httpx.Response(200, json=[[0.0] * 768]) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 768 + + +def test_fetch_dimensions_probe_fallback_when_info_denied(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(401) + if request.url.path == "/embed": + return httpx.Response(200, json=[[0.0] * 384]) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 384 + + +async def test_auth_header_sent_when_api_key_set(): + seen: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers.get("authorization", "") + return httpx.Response(200, json=[[0.0] * 4]) + + client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + base_url="http://tei.test", + headers={"Authorization": "Bearer sekret"}, + ) + provider = TEIEmbeddingProvider( + url="http://tei.test", api_key="sekret", _client=client + ) + await provider.embed_query("q") + assert seen["auth"] == "Bearer sekret" + await provider.aclose() + + +async def test_embed_raises_on_server_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + provider = _make_provider(handler) + with pytest.raises(httpx.HTTPStatusError): + await provider.embed_documents(["doc"]) + await provider.aclose() diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index d7cc1c4a..894a0030 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -88,6 +88,16 @@ class EmbeddingConfig(BaseSettings): alias="VEKTRA_SPARSE_EMBEDDING_MODEL", description="Sparse embedding model name. Phase 2 only.", ) + tei_url: str = Field( + "http://localhost:8080", + alias="VEKTRA_TEI_URL", + description="TEI server base URL (native API, no /v1 suffix). Used when embedding_provider='tei' (FEAT-024).", + ) + tei_api_key: str | None = Field( + None, + alias="VEKTRA_TEI_API_KEY", + description="Bearer token for the TEI embedding server (--api-key). Optional.", + ) model_config = SettingsConfigDict( env_prefix="", extra="ignore", populate_by_name=True @@ -158,7 +168,7 @@ class RerankConfig(BaseSettings): provider: str = Field( "cross-encoder", alias="VEKTRA_RERANK_PROVIDER", - description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere'.", + description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere', 'tei'.", ) model: str | None = Field( "BAAI/bge-reranker-v2-m3", @@ -171,6 +181,21 @@ class RerankConfig(BaseSettings): alias="VEKTRA_RERANK_TOP_K", description="Final top-k results after reranking.", ) + api_key: str | None = Field( + None, + alias="VEKTRA_RERANK_API_KEY", + description="API key for API-based rerank providers (e.g. 'cohere').", + ) + tei_url: str = Field( + "http://localhost:8080", + alias="VEKTRA_RERANK_TEI_URL", + description="TEI reranker server base URL (one TEI instance per model). Used when provider='tei' (FEAT-024).", + ) + tei_api_key: str | None = Field( + None, + alias="VEKTRA_RERANK_TEI_API_KEY", + description="Bearer token for the TEI reranker server. Optional.", + ) model_config = SettingsConfigDict( env_prefix="", extra="ignore", populate_by_name=True @@ -499,6 +524,8 @@ class VektraSettings(BaseSettings): sparse_embedding_model: str | None = Field( None, alias="VEKTRA_SPARSE_EMBEDDING_MODEL" ) + tei_url: str = Field("http://localhost:8080", alias="VEKTRA_TEI_URL") + tei_api_key: str | None = Field(None, alias="VEKTRA_TEI_API_KEY") # Vector store vector_store_provider: str = Field("pgvector", alias="VEKTRA_VECTOR_STORE_PROVIDER") From 5cca19cf40be525a4a7f144ef5be169e1038444e Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 01:49:40 +0000 Subject: [PATCH 28/31] fix(index): look up the active embedding provider in the startup warmup check_embedding_model resolved the provider by the hardcoded 'sentence-transformers' name, so startup failed with a registration error whenever VEKTRA_EMBEDDING_PROVIDER=tei was active (found in the FEAT-024 live smoke). The 'default' alias always points to the active provider. Co-Authored-By: Claude Fable 5 --- vektra-index/src/vektra_index/startup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vektra-index/src/vektra_index/startup.py b/vektra-index/src/vektra_index/startup.py index 1ffd8575..ab4a6d2c 100644 --- a/vektra-index/src/vektra_index/startup.py +++ b/vektra-index/src/vektra_index/startup.py @@ -52,7 +52,9 @@ async def check_provider_registration( async def check_embedding_model(registry: Any) -> None: """ARCH-057 step 6: warm up the embedding model and verify dimensionality.""" try: - embedding_provider = registry.get("embedding", "sentence-transformers") + # "default" aliases whichever provider is active + # (sentence-transformers or tei, FEAT-024). + embedding_provider = registry.get("embedding", "default") test_embedding = await embedding_provider.embed_query("startup validation test") actual_dims = len(test_embedding) expected_dims = embedding_provider.dimensions() From 390ddcf23eab13876d74b012534cc9c858f7fa60 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 02:16:59 +0000 Subject: [PATCH 29/31] docs(backlog): mark FEAT-024 completed with comparison numbers bge-m3 via TEI on the eval-full questions (same dual chunks): retrieval hit 82.6% -> 93.5%, MRR 0.7029 -> 0.8478, e2e stable. Configuration reference and .env.example cover the new provider options; the reindex pgvector-hardcoding discovery is recorded in the BUG-021 family note. Co-Authored-By: Claude Fable 5 --- .env.example | 8 ++++++++ .s2s/BACKLOG.md | 17 ++++++++++------- .s2s/plans/20260712-sprint3-rag-quality.md | 22 ++++++++++++++++++++++ docs/reference/configuration.md | 11 ++++++++--- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index e576fa2d..01d93810 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,9 @@ # Embedding provider and model. # VEKTRA_EMBEDDING_PROVIDER=sentence-transformers # VEKTRA_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 +# Remote embedding via Text Embeddings Inference (VEKTRA_EMBEDDING_PROVIDER=tei): +# VEKTRA_TEI_URL=http://localhost:8080 +# VEKTRA_TEI_API_KEY= # -------------------------------------------------------------------------- # Vector store @@ -106,6 +109,11 @@ # VEKTRA_RERANK_PROVIDER=cross-encoder # VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 # VEKTRA_RERANK_TOP_K=5 +# API key for API-based providers (cohere): +# VEKTRA_RERANK_API_KEY= +# Remote reranking via TEI (VEKTRA_RERANK_PROVIDER=tei, one instance per model): +# VEKTRA_RERANK_TEI_URL=http://localhost:8080 +# VEKTRA_RERANK_TEI_API_KEY= # -------------------------------------------------------------------------- # Ingestion diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 3bd8e9fb..c8f9feca 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -393,8 +393,9 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. ### FEAT-024: Remote embedding and reranker providers (TEI) -**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Status**: completed | **Priority**: medium | **Created**: 2026-07-12 | **Completed**: 2026-07-13 | **PR**: #93 **Origin**: deployment modularity review 2026-07-12 - the host workstation already serves TEI instances (bge-m3, qwen3-embedding); Vektra cannot use them. +**Evidence**: `vektra-internal/stack/20260713-feat024-tei-providers.md` (+ artifacts in `20260713-feat024-eval-artifacts/`) **Context**: `VEKTRA_EMBEDDING_PROVIDER` documents a `tei` option (config.py:74) but **no TEI provider exists**: `main.py:129-137` unconditionally instantiates in-process `SentenceTransformersProvider`; the compose even ships a `tei` profile service nobody can talk to. The reranker likewise runs in-process only (`rerankers` lib; the `cohere` path never passes an api_key, so it is dead as wired - reranker.py:122). Consequences: every Vektra instance duplicates embedding/reranker compute in-container (CPU), and shared GPU/CPU inference services on the host cannot be reused. @@ -406,14 +407,16 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. **Why it matters beyond dedup**: the current embedding model (paraphrase-multilingual-MiniLM-L12-v2) has **max_seq_length 128 tokens** - our 500-token chunks are silently truncated at embedding time (dense sees only the chunk head; BM25 sees the full text). bge-m3 (8192-token window, MIRACL dense nDCG@10 69.2 vs mE5-large 66.6; MiniLM sits 16-22 nDCG points below even mE5 on European-language retrieval per PL-MTEB) is the natural upgrade candidate, testable via TEI without fattening the container. +**Resolution (2026-07-13)**: implemented as designed with two deltas: (1) TEI 1.9.3 `/info` does not expose the embedding size, so `dimensions()` probes `/embed` as fallback (both paths unit-tested); (2) found and fixed in passing a startup blocker: `check_embedding_model` resolved the provider by the hardcoded `sentence-transformers` name, so startup failed with any other provider (now uses the `default` alias). Measured on eval-full questions (same dual chunks, reindexed via a fresh `eval-tei` namespace): **bge-m3 via TEI retrieval hit 93.5% / MRR 0.8478 vs MiniLM dual 82.6% / 0.7029 (+10.9pp)** - beats even the fixed-chunking MiniLM baseline (89.1%/0.8062), confirming the 128-token truncation hypothesis; e2e grounded 54/55 stable, MC answers improve in substance (MC-02 produces a real bi-document comparison; kw 7/25 vs 4/25), p50 +0.7s (TEI on CPU). TEI reranker smoke: factual query scores 0.75 (2 survive the threshold), comparative query all-below-threshold rescued by TECH-007 (`rescued=3`) - full funnel verified with both remote providers, authenticated. Discovery filed under BUG-021: `run_reindex` stores through hardcoded pgvector, so reindex-into-Qdrant silently writes nothing (worked around via fresh-namespace ingest). Switching the default embedding to bge-m3 is a separate decision (needs full corpus re-ingest and a TECH-005-grade bench). + **Traceability**: ADR-0013 (EmbeddingProvider Protocol), ARCH-035, ARCH-036, ADR-0021 **Acceptance criteria**: -- [ ] `VEKTRA_EMBEDDING_PROVIDER=tei` works end-to-end (ingest + query) against a TEI instance with api key -- [ ] Collection created with the provider's real dimensions; clear error on dimension mismatch with an existing collection -- [ ] `VEKTRA_RERANK_PROVIDER=tei` reranks via TEI /rerank with scores compatible with the threshold filter -- [ ] Embedding-model comparison (MiniLM in-process vs bge-m3 via TEI) run with the TECH-005/existing harness and recorded -- [ ] Docs: configuration.md + .env.example cover the new provider options +- [x] `VEKTRA_EMBEDDING_PROVIDER=tei` works end-to-end (ingest + query) against a TEI instance with api key +- [x] Collection created with the provider's real dimensions; clear error on dimension mismatch with an existing collection (verified live: 384-vs-1024 startup warning with remediation) +- [x] `VEKTRA_RERANK_PROVIDER=tei` reranks via TEI /rerank with scores compatible with the threshold filter +- [x] Embedding-model comparison (MiniLM in-process vs bge-m3 via TEI) run with the TECH-005/existing harness and recorded +- [x] Docs: configuration.md + .env.example cover the new provider options --- @@ -1087,7 +1090,7 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou **Resolution**: the endpoint now resolves embedding, sparse embedding, and vector store from `request.app.state.registry` (same contract as the pipeline). The per-request `SentenceTransformersProvider` instantiation and the now-unused `session` dependency were removed. Unit tests added (`vektra-index/tests/test_api_search.py`): registry resolution, hybrid→dense fallback without sparse, hybrid with sparse. -**Same family, not fixed here**: `POST /documents/{id}/chunks`, `DELETE /documents/{id}` and `GET /stats` still hardcode pgvector (the stats-vs-Qdrant mismatch was already a known issue). Track separately if needed. +**Same family, not fixed here**: `POST /documents/{id}/chunks`, `DELETE /documents/{id}` and `GET /stats` still hardcode pgvector (the stats-vs-Qdrant mismatch was already a known issue). Also `run_reindex` (`vektra-index/reindex.py`): it re-embeds with the registry's active embedding provider but stores through a hardcoded `PgvectorProvider`, so in Qdrant mode a reindex reports "completed" while the Qdrant collection receives nothing (found during the FEAT-024 live smoke, 2026-07-13: reindexing eval-full to v2 wrote to Postgres only). Track separately if needed. **Traceability**: ARCH-039 (ProviderRegistry), ARCH-051 (full-store contract), TECH-002 (eval harness) diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md index 86ae3f93..938f9de4 100644 --- a/.s2s/plans/20260712-sprint3-rag-quality.md +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -323,3 +323,25 @@ the bundle and a manual smoke in the Moodle dev stack. `vektra-internal/stack/20260713-tech007-retrieval-rescue.md`; per-question artifacts in `20260713-tech007-eval-artifacts/`. Dev `.env` adds `VEKTRA_RETRIEVAL_RESCUE_TOP_K=3`, `VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.005`. +- 2026-07-13: **FEAT-024 implemented and measured (post-sprint, PR #93)** — + TEI remote providers, `feat/feat-024-tei-providers`. TEIEmbeddingProvider + (native /embed, /info dimensions with /embed probe fallback — TEI 1.9.3 + does not expose the size in /info), TEIRerankerService (/rerank, same + RerankResult semantics), Qdrant collection sized from the active provider + (fixes the hardcoded-384 latent bug; live 384-vs-1024 mismatch produces a + clear startup warning), cohere api_key pass-through fixed, startup warmup + fixed to resolve the `default` embedding alias (was hardcoded to + 'sentence-transformers' and broke any alternative provider). + **Comparison on eval-full questions (same dual chunks, ns eval-tei, + bge-m3 float32 CPU via TEI): retrieval hit 82.6% -> 93.5%, MRR 0.7029 -> + 0.8478 — beats even the fixed-chunking MiniLM baseline (89.1%/0.8062), + confirming MiniLM's 128-token truncation as a real retrieval cap.** E2e + grounded 54/55 stable, multi-chunk substance improves (MC-02 real + bi-document comparison, kw 7/25 vs 4/25), p50 +0.7s (CPU TEI). Reranker + smoke: factual 0.75 survives threshold, comparative all-below rescued by + TECH-007 (rescued=3) — full funnel verified with both remote providers. + Found in passing (filed under BUG-021 family): run_reindex stores through + hardcoded pgvector — reindex-into-Qdrant silently no-ops. Dev stack + restored to reference config (MiniLM + rescue); eval-tei namespace and + vektra-tei collection (105 points, 1024-dim) left in place for follow-ups; + bge-reranker TEI cache kept at /mnt/scratch/tei-rerank-cache. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1a50360d..72a32279 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -59,8 +59,10 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| -| `VEKTRA_EMBEDDING_PROVIDER` | str | `sentence-transformers` | Embedding provider implementation | -| `VEKTRA_EMBEDDING_MODEL` | str | `paraphrase-multilingual-MiniLM-L12-v2` | Model name within the selected provider | +| `VEKTRA_EMBEDDING_PROVIDER` | str | `sentence-transformers` | Embedding provider implementation: `sentence-transformers` (in-process), `tei` (remote) | +| `VEKTRA_EMBEDDING_MODEL` | str | `paraphrase-multilingual-MiniLM-L12-v2` | Model name within the selected provider (`sentence-transformers` only; a TEI instance serves one fixed model) | +| `VEKTRA_TEI_URL` | str | `http://localhost:8080` | TEI server base URL (native API, no `/v1` suffix). Used when provider is `tei`. The Qdrant collection is sized from the served model's dimensions at startup | +| `VEKTRA_TEI_API_KEY` | str | - | Bearer token for the TEI embedding server (`--api-key`). Optional | | `VEKTRA_SPARSE_EMBEDDING_PROVIDER` | str | - | Sparse embedding provider: `fastembed-bm25`, `splade` | | `VEKTRA_SPARSE_EMBEDDING_MODEL` | str | - | Sparse embedding model name | @@ -101,9 +103,12 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_RERANK_ENABLED` | bool | `true` | Enable cross-encoder reranking after retrieval | -| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | +| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere`, `tei` (remote) | | `VEKTRA_RERANK_MODEL` | str | `BAAI/bge-reranker-v2-m3` | Multilingual reranking model. For English-only lightweight deployments: provider=`flashrank`, model=`ms-marco-MiniLM-L-12-v2` | | `VEKTRA_RERANK_TOP_K` | int | `5` | Final top-k results after reranking | +| `VEKTRA_RERANK_API_KEY` | str | - | API key for API-based providers (`cohere`) | +| `VEKTRA_RERANK_TEI_URL` | str | `http://localhost:8080` | TEI reranker server base URL (one TEI instance per model, e.g. serving `BAAI/bge-reranker-v2-m3`). Used when provider is `tei`. Scores are sigmoid-normalized like the in-process path | +| `VEKTRA_RERANK_TEI_API_KEY` | str | - | Bearer token for the TEI reranker server. Optional | ## Ingestion From dfbc961bab40b8167f96041208d8ff6ca2d257cc Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 02:32:49 +0000 Subject: [PATCH 30/31] fix(rag): harden TEI providers per review - create_reranker: the tei branch now degrades to None on init/URL errors like the in-process path, and logs the URL redacted (coderabbit #3567707446, #3567707458) - TEIRerankerService gains aclose(); the app lifespan teardown closes remote provider clients best-effort via a new reranker registry entry (coderabbit #3567707442, #3567707464) - _embed_batch warms the dimensions cache so the startup warmup makes the synchronous dimensions() call free (coderabbit #3567707461) - _fetch_dimensions tolerates non-JSON /info responses and falls back to the /embed probe (gemini #3567695402) - ensure_collection reads the dense vector params via getattr fallback for non-dict vector configs (gemini #3567695404) Co-Authored-By: Claude Fable 5 --- vektra-app/src/vektra_app/main.py | 15 ++++++++++ vektra-core/src/vektra_core/reranker.py | 24 ++++++++++++++-- vektra-core/tests/test_reranker.py | 10 +++++++ .../src/vektra_index/providers/qdrant.py | 6 +++- .../src/vektra_index/providers/tei.py | 28 +++++++++++-------- 5 files changed, 68 insertions(+), 15 deletions(-) diff --git a/vektra-app/src/vektra_app/main.py b/vektra-app/src/vektra_app/main.py index f112f9a8..30e8f1bb 100644 --- a/vektra-app/src/vektra_app/main.py +++ b/vektra-app/src/vektra_app/main.py @@ -267,6 +267,9 @@ async def _step_5_register_providers( pipeline_config = QueryPipelineConfig() reranker = create_reranker(pipeline_config.rerank) + if reranker is not None: + # Registered so the lifespan teardown can close remote clients (FEAT-024) + registry.register("reranker", "default", reranker) # --- Query pipeline (Phase 2: select simple or advanced) --- templates_dir = ( @@ -559,6 +562,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Shutdown log.info("shutdown_started") + # Close remote provider HTTP clients (TEI, FEAT-024) best-effort + for category in ("embedding", "reranker"): + try: + provider = registry.get(category, "default") + except Exception: + continue + aclose = getattr(provider, "aclose", None) + if aclose is not None: + try: + await aclose() + except Exception as exc: + log.warning("provider_close_failed", category=category, error=str(exc)) from vektra_shared.db import get_engine engine = get_engine() diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index a8cde2bf..eba02e74 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -15,6 +15,7 @@ import dataclasses import math from typing import Protocol, runtime_checkable +from urllib.parse import urlparse import httpx import structlog @@ -32,6 +33,14 @@ } +def _redact_url(url: str) -> str: + """Return scheme://hostname:port only, stripping credentials and path.""" + parsed = urlparse(url) + host = parsed.hostname or "" + port = f":{parsed.port}" if parsed.port else "" + return f"{parsed.scheme}://{host}{port}" + + def _sigmoid(x: float) -> float: """Numerically stable sigmoid for cross-encoder logits.""" if x >= 0: @@ -176,6 +185,10 @@ async def rerank( ordered = [(int(item["index"]), float(item["score"])) for item in ranked] return _build_rerank_result(results, ordered, top_k) + async def aclose(self) -> None: + """Release the underlying HTTP client (tests and shutdown).""" + await self._client.aclose() + def create_reranker(config: RerankConfig) -> RerankerProtocol | None: """Create a reranker service from config. Returns None if unavailable.""" @@ -184,8 +197,15 @@ def create_reranker(config: RerankConfig) -> RerankerProtocol | None: return None if config.provider == "tei": - log.info("reranker_loaded", provider="tei", url=config.tei_url) - return TEIRerankerService(url=config.tei_url, api_key=config.tei_api_key) + try: + service = TEIRerankerService(url=config.tei_url, api_key=config.tei_api_key) + log.info("reranker_loaded", provider="tei", url=_redact_url(config.tei_url)) + except Exception as exc: + # Same graceful-degradation contract as the in-process path: + # a bad reranker config must not abort startup. + log.warning("reranker_init_failed", provider="tei", error=str(exc)) + return None + return service model_type = _PROVIDER_TO_MODEL_TYPE.get(config.provider, config.provider) model_name = config.model or _default_model_for_provider(config.provider) diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index 59ef0db6..9621653a 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -317,3 +317,13 @@ def test_create_reranker_tei_provider(): ) reranker = create_reranker(config) assert isinstance(reranker, TEIRerankerService) + + +def test_create_reranker_tei_invalid_url_returns_none(): + """A malformed TEI URL degrades to None instead of aborting startup.""" + config = RerankConfig( + VEKTRA_RERANK_ENABLED=True, + VEKTRA_RERANK_PROVIDER="tei", + VEKTRA_RERANK_TEI_URL="http://[invalid", + ) + assert create_reranker(config) is None diff --git a/vektra-index/src/vektra_index/providers/qdrant.py b/vektra-index/src/vektra_index/providers/qdrant.py index 71dbf39e..9e073c16 100644 --- a/vektra-index/src/vektra_index/providers/qdrant.py +++ b/vektra-index/src/vektra_index/providers/qdrant.py @@ -118,7 +118,11 @@ async def ensure_collection(self) -> None: if self._collection_name in existing: info = await self._client.get_collection(self._collection_name) vectors = info.config.params.vectors - dense = vectors.get("dense") if isinstance(vectors, dict) else None + dense = ( + vectors.get("dense") + if isinstance(vectors, dict) + else getattr(vectors, "dense", None) + ) existing_size = getattr(dense, "size", None) if existing_size is not None and existing_size != self._dense_dimensions: raise ValueError( diff --git a/vektra-index/src/vektra_index/providers/tei.py b/vektra-index/src/vektra_index/providers/tei.py index 75f2b165..ffead977 100644 --- a/vektra-index/src/vektra_index/providers/tei.py +++ b/vektra-index/src/vektra_index/providers/tei.py @@ -57,6 +57,10 @@ async def _embed_batch(self, texts: list[str]) -> list[list[float]]: resp = await self._client.post("/embed", json={"inputs": texts}) resp.raise_for_status() data: list[list[float]] = resp.json() + if data and self._dimensions is None: + # Warm the dimensions cache so the startup warmup (embed_query) + # makes the later synchronous dimensions() call free. + self._dimensions = len(data[0]) return data async def embed_documents(self, texts: list[str]) -> list[list[float]]: @@ -93,18 +97,18 @@ def dimensions(self) -> int: @staticmethod def _fetch_dimensions(client: httpx.Client) -> int: try: - info: dict[str, Any] = client.get("/info").raise_for_status().json() - for key in ("embedding_size", "hidden_size"): - if isinstance(info.get(key), int): - return int(info[key]) - except httpx.HTTPStatusError: - logger.debug("TEI /info unavailable, probing /embed for dimensions") - probe = ( - client.post("/embed", json={"inputs": ["dim probe"]}) - .raise_for_status() - .json() - ) - return len(probe[0]) + resp = client.get("/info") + resp.raise_for_status() + info: Any = resp.json() + if isinstance(info, dict): + for key in ("embedding_size", "hidden_size"): + if isinstance(info.get(key), int): + return int(info[key]) + except (httpx.HTTPError, ValueError): + logger.debug("TEI /info unavailable or invalid, probing /embed") + resp = client.post("/embed", json={"inputs": ["dim probe"]}) + resp.raise_for_status() + return len(resp.json()[0]) async def health_check(self) -> HealthStatus: """Verify the TEI server responds and can produce an embedding.""" From bf22fff1433b8e2df453729ff6ccdaf62c1f9c97 Mon Sep 17 00:00:00 2001 From: Francesco Vadicamo Date: Mon, 13 Jul 2026 02:38:49 +0000 Subject: [PATCH 31/31] chore(release): finalize v0.6.0 - Bump version 0.5.1 -> 0.6.0 across 8 components - Promote CHANGELOG entries from [Unreleased] to [0.6.0] - 2026-07-13 (FEAT-017 parent expansion, TECH-007 retrieval rescue, FEAT-021 citations, FEAT-024 TEI providers, BUG-021/BUG-022, DEBT-025) - Align vektra_app.__version__ (was stuck at 0.4.0-dev since v0.5.0; surfaced by /health and the OpenAPI spec) - Refresh uv.lock for the version change Prepares the develop -> main release PR for v0.6.0. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 ++++ uv.lock | 16 ++++++++-------- vektra-admin/pyproject.toml | 2 +- vektra-analytics/pyproject.toml | 2 +- vektra-app/pyproject.toml | 2 +- vektra-app/src/vektra_app/__init__.py | 2 +- vektra-core/pyproject.toml | 2 +- vektra-index/pyproject.toml | 2 +- vektra-ingest/pyproject.toml | 2 +- vektra-learn/pyproject.toml | 2 +- vektra-shared/pyproject.toml | 2 +- 11 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e719f54..1122283a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Convention (Keep a Changelog 1.1.0): +## [0.6.0] - 2026-07-13 + +RAG quality release: parent chunk expansion, retrieval-filter rescue, per-namespace citations, remote TEI providers. + ### Added - **rag**: remote embedding and reranking via HuggingFace Text Embeddings Inference (FEAT-024). `VEKTRA_EMBEDDING_PROVIDER=tei` embeds through a TEI instance (`VEKTRA_TEI_URL`/`VEKTRA_TEI_API_KEY`, native `/embed` API) instead of in-process sentence-transformers, enabling shared host inference and long-window models (bge-m3: 8192 tokens vs MiniLM's 128, which silently truncates 500-token chunks today). `VEKTRA_RERANK_PROVIDER=tei` reranks through TEI `/rerank` (`VEKTRA_RERANK_TEI_URL`/`VEKTRA_RERANK_TEI_API_KEY`). The Qdrant collection is now sized from the active embedding provider's dimensions instead of a hardcoded 384 (latent bug for any non-384 model), with a clear startup error on dimension mismatch against an existing collection. The `cohere` rerank option now actually passes its API key (`VEKTRA_RERANK_API_KEY`); it was dead as wired. diff --git a/uv.lock b/uv.lock index 112aae6f..c7dff6be 100644 --- a/uv.lock +++ b/uv.lock @@ -5099,7 +5099,7 @@ wheels = [ [[package]] name = "vektra-admin" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-admin" } dependencies = [ { name = "argon2-cffi" }, @@ -5148,7 +5148,7 @@ dev = [ [[package]] name = "vektra-analytics" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-analytics" } dependencies = [ { name = "fastapi" }, @@ -5183,7 +5183,7 @@ dev = [ [[package]] name = "vektra-app" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-app" } dependencies = [ { name = "alembic" }, @@ -5232,7 +5232,7 @@ dev = [ [[package]] name = "vektra-core" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-core" } dependencies = [ { name = "fastapi" }, @@ -5277,7 +5277,7 @@ dev = [ [[package]] name = "vektra-index" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-index" } dependencies = [ { name = "asyncpg" }, @@ -5334,7 +5334,7 @@ dev = [ [[package]] name = "vektra-ingest" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-ingest" } dependencies = [ { name = "arq" }, @@ -5399,7 +5399,7 @@ dev = [ [[package]] name = "vektra-learn" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-learn" } dependencies = [ { name = "fastapi" }, @@ -5436,7 +5436,7 @@ dev = [ [[package]] name = "vektra-shared" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-shared" } dependencies = [ { name = "asyncpg" }, diff --git a/vektra-admin/pyproject.toml b/vektra-admin/pyproject.toml index 84be6c56..70c64737 100644 --- a/vektra-admin/pyproject.toml +++ b/vektra-admin/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-admin" -version = "0.5.1" +version = "0.6.0" description = "System administration interface: health, API key management, audit log" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-analytics/pyproject.toml b/vektra-analytics/pyproject.toml index d08dd8e2..7e97aee9 100644 --- a/vektra-analytics/pyproject.toml +++ b/vektra-analytics/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-analytics" -version = "0.5.1" +version = "0.6.0" description = "QueryTrace storage, metrics aggregation, and reporting API" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-app/pyproject.toml b/vektra-app/pyproject.toml index f3e42aac..a4b2bde3 100644 --- a/vektra-app/pyproject.toml +++ b/vektra-app/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-app" -version = "0.5.1" +version = "0.6.0" description = "FastAPI application assembly and startup validation (ARCH-015, ARCH-057)" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-app/src/vektra_app/__init__.py b/vektra-app/src/vektra_app/__init__.py index 5f0ed55f..ea690d8e 100644 --- a/vektra-app/src/vektra_app/__init__.py +++ b/vektra-app/src/vektra_app/__init__.py @@ -1,3 +1,3 @@ # vektra-app: FastAPI assembly and startup validation (ARCH-015) -__version__ = "0.4.0-dev" +__version__ = "0.6.0" diff --git a/vektra-core/pyproject.toml b/vektra-core/pyproject.toml index b8259147..5946fbfe 100644 --- a/vektra-core/pyproject.toml +++ b/vektra-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-core" -version = "0.5.1" +version = "0.6.0" description = "RAG engine, LLM abstraction, and conversation management" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-index/pyproject.toml b/vektra-index/pyproject.toml index 9bcb0cc9..91084b62 100644 --- a/vektra-index/pyproject.toml +++ b/vektra-index/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-index" -version = "0.5.1" +version = "0.6.0" description = "Vector store abstraction and semantic search for the Vektra platform" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-ingest/pyproject.toml b/vektra-ingest/pyproject.toml index cc153eac..15e9e8ad 100644 --- a/vektra-ingest/pyproject.toml +++ b/vektra-ingest/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-ingest" -version = "0.5.1" +version = "0.6.0" description = "Document processing pipeline for the Vektra platform" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-learn/pyproject.toml b/vektra-learn/pyproject.toml index 41866965..c9728987 100644 --- a/vektra-learn/pyproject.toml +++ b/vektra-learn/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-learn" -version = "0.5.1" +version = "0.6.0" description = "E-learning vertical: LMS-agnostic API and chatbot widget" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-shared/pyproject.toml b/vektra-shared/pyproject.toml index 1e56bd84..1e125d54 100644 --- a/vektra-shared/pyproject.toml +++ b/vektra-shared/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-shared" -version = "0.5.1" +version = "0.6.0" description = "Shared protocols and types for the Vektra platform" readme = "README.md" requires-python = ">=3.12"