diff --git a/.env.example b/.env.example index 5cdbec2f..01d93810 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,9 @@ # Embedding provider and model. # VEKTRA_EMBEDDING_PROVIDER=sentence-transformers # VEKTRA_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 +# Remote embedding via Text Embeddings Inference (VEKTRA_EMBEDDING_PROVIDER=tei): +# VEKTRA_TEI_URL=http://localhost:8080 +# VEKTRA_TEI_API_KEY= # -------------------------------------------------------------------------- # Vector store @@ -84,6 +87,9 @@ # VEKTRA_QUERY_PIPELINE=advanced # VEKTRA_MIN_RELEVANCE_SCORE=0.15 # VEKTRA_CHUNK_DEDUP_ENABLED=true +# VEKTRA_RETRIEVAL_RESCUE_TOP_K=0 +# VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.02 +# VEKTRA_PARENT_EXPANSION_ENABLED=false # VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 # VEKTRA_PROMPT_TEMPLATES_DIR= @@ -103,6 +109,11 @@ # VEKTRA_RERANK_PROVIDER=cross-encoder # VEKTRA_RERANK_MODEL=BAAI/bge-reranker-v2-m3 # VEKTRA_RERANK_TOP_K=5 +# API key for API-based providers (cohere): +# VEKTRA_RERANK_API_KEY= +# Remote reranking via TEI (VEKTRA_RERANK_PROVIDER=tei, one instance per model): +# VEKTRA_RERANK_TEI_URL=http://localhost:8080 +# VEKTRA_RERANK_TEI_API_KEY= # -------------------------------------------------------------------------- # Ingestion diff --git a/.github/workflows/docker-ocr-build.yml b/.github/workflows/docker-ocr-build.yml new file mode 100644 index 00000000..9437aa8b --- /dev/null +++ b/.github/workflows/docker-ocr-build.yml @@ -0,0 +1,36 @@ +# Guard for the INSTALL_UNSTRUCTURED image variant (BUG-022). +# The OCR extra pulls torch-adjacent wheels whose index pinning can silently +# break the build; regular CI never builds with the flag, so this workflow +# does, but only when the inputs that can break it change. +name: docker-ocr-build + +on: + pull_request: + paths: + - "Dockerfile" + - "uv.lock" + - "pyproject.toml" + - "vektra-ingest/pyproject.toml" + - ".github/workflows/docker-ocr-build.yml" + push: + branches: [develop, main] + paths: + - "Dockerfile" + - "uv.lock" + - "pyproject.toml" + - "vektra-ingest/pyproject.toml" + - ".github/workflows/docker-ocr-build.yml" + +permissions: + contents: read + +jobs: + build-ocr-image: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Build image with INSTALL_UNSTRUCTURED=true + run: docker build --build-arg INSTALL_UNSTRUCTURED=true -t vektra-ocr-ci . diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index ba87d95b..c8f9feca 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -65,8 +65,15 @@ The LLM already has native conversational coherence: it sees the full history an ### FEAT-018: Exclude previously retrieved chunks in multi-turn conversations -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 -**Depends on**: evaluate BUG-020 option 1 (prompt fix) first - this may not be needed if the prompt change resolves multi-turn coherence. +**Status**: deferred (2026-07-12, verified) | **Priority**: low | **Created**: 2026-03-28 +**Depends on**: TECH-007 (rerank+threshold funnel) - with the current funnel, excluding previously seen chunks would increase multi-turn refusals, not variety. + +**Verification (2026-07-12, Sprint 3, plan `20260712-sprint3-rag-quality` section 4)**: 4 multi-turn scenarios run against `/api/v1/query` with `conversation_id` on the `eval-full` corpus (rewrite + FEAT-020 history + FEAT-017 expansion active, eval-mode traces inspected): +- **Topic switch** and **negation**: already fully mitigated - the rewriter produces clean standalone queries, retrieved chunks change completely (0 shared), reranker scores high (0.99 / 0.82). No exclusion needed. +- **Same-topic follow-up** ("quali limiti prevede l'art. 21?"): rewrite is correct but the turn dies at the retrieval filter (max rerank 0.093 < 0.15) - a TECH-007 failure, which chunk exclusion would make worse, not better. +- **"Give me more"**: the one genuine FEAT-018 case. Rewrite is explicit ("altri diritti... diversi da quelli gia' citati") yet the prompt receives the same 3 chunks as turn 1 (3/3 shared); the LLM degrades gracefully via history (acknowledges prior answer, avoids verbatim repetition) but cannot produce genuinely new content from identical material. + +**Decision**: no-go for Sprint 3. Revisit only after TECH-007 lands (a funnel that admits more chunks makes exclusion safe and useful), with "give me more" as the driving scenario and the original design below. **Context**: in multi-turn conversations, the vector search returns the same high-scoring chunks every turn, even when the user explicitly asks for "other" or "different" results. The query rewrite contextualizes the question but the retrieval still matches on semantic similarity, which favors the same chunks. @@ -223,7 +230,8 @@ Rules: ### FEAT-021: Optional source citations in responses (per-namespace) -**Status**: planned | **Priority**: medium | **Created**: 2026-03-28 +**Status**: completed (2026-07-12) | **Priority**: medium | **Created**: 2026-03-28 +**Resolution**: shipped on `feat/feat-021-namespace-citations` (Sprint 3, plan `20260712-sprint3-rag-quality` section 5). Deltas vs the design below: the context template keeps the existing `` tag (the `` sketch predates the template); `SearchResult` gains no new fields (source_file/page were already in `metadata`; the title is composed at prompt-build time from `_fetch_document_names` + page); resolution is namespace JSONB > hardcoded false, no env var; the widget renders `[n]` as superscript with the source title as native tooltip, wired in `addSources()`. Default-off renders byte-identical prompts (`prompt_version` changes because the template files changed). **Context**: in some deployment contexts (academic research, compliance, legal), full transparency with source citations is required. Currently Rule 1 in the system prompt forbids any mention of sources ("Never mention, quote, or allude to sources, documents, context tags, or reference material"). This is correct for the default e-learning use case where the student should not know about the RAG pipeline, but must be optional for contexts where traceability is a requirement. @@ -271,31 +279,161 @@ The `title` field would contain `filename + page` (e.g., "Costituzione italiana. **Traceability**: ARCH-054 (composable templates), ARCH-047 (namespace metadata), ADR-0025 (chatbot widget) **Acceptance criteria**: -- [ ] `citations_enabled` per-namespace setting in namespace metadata JSONB -- [ ] `system.j2` Rule 1 conditional: cite with `[id]` when enabled, hide sources when disabled -- [ ] `context.j2` includes document title in `` elements when citations enabled -- [ ] Document filename and page propagated through `SearchResult` to template -- [ ] `SourceRef` includes `title` field in API response -- [ ] Widget renders `[id]` references as tooltips or footnotes with source info -- [ ] Default behavior unchanged (citations disabled, Rule 1 hides sources) +- [x] `citations_enabled` per-namespace setting in namespace metadata JSONB (admin PATCH whitelist + GET resolved defaults) +- [x] `system.j2` Rule 1 conditional: cite with `[id]` when enabled (and context present), hide sources when disabled +- [x] `context.j2` includes document title in `` elements when citations enabled +- [x] Document filename and page propagated to the template (composed at prompt-build time from `_fetch_document_names` + `metadata.page`) +- [x] `SourceRef` includes `title` field in API response (core + learn, sync + streaming) +- [x] Widget renders `[id]` references as superscripts with source-title tooltips +- [x] Default behavior unchanged (citations disabled, Rule 1 hides sources; byte-identical rendering asserted in tests) --- ### FEAT-017: Parent chunk expansion in query pipeline -**Status**: planned | **Priority**: medium | **Created**: 2026-03-23 +**Status**: completed (2026-07-12) | **Priority**: medium | **Created**: 2026-03-23 **Analysis**: `vektra-internal/stack/20260323-rag-prompt-chunk-confusion-analysis.md` **Context**: when a child chunk is retrieved via search, the pipeline should optionally expand it to the parent chunk for broader context. The infrastructure is already in place: `DualStrategyChunking` creates parent-child hierarchy (parent every 3000 tokens, children at 500 tokens with overlap), `DocumentChunkOrm` has `parent_id` column, and both are stored in the database. Missing: (1) filter parent chunks from default search results (search currently returns both), (2) parent expansion logic in AdvancedQueryPipeline when a child matches. +**Premise corrections found during implementation**: the hierarchy was NOT actually persisted anywhere (`run_ingest` dropped `chunk.parent_id`, `ChunkEmbedding` had no field, Qdrant payload had none, `DocumentChunkOrm.parent_id` was always NULL); parent size is `chunk_size*3` (1500 tokens with Combo D 500, not 3000); pgvector `store()` ignored caller chunk ids (generated uuid4), so deterministic ids had to be plumbed there too. + **Traceability**: ARCH-037 (ChunkingStrategy), ARCH-055 (token budget), core-pipeline-v2 **Acceptance criteria**: -- [ ] Search excludes parent chunks by default (WHERE parent_id IS NOT NULL for children only) -- [ ] AdvancedQueryPipeline fetches parent chunk when child matches and includes it in context -- [ ] Parent expansion is configurable (on/off, via env var) -- [ ] Token budget accounts for expanded parent chunk size -- [ ] Tested: truncated-context answers improve with parent expansion enabled +- [x] Search excludes parent chunks by default (`chunk_level=parent` filtered: Qdrant `must_not`, pgvector `IS DISTINCT FROM`) +- [x] AdvancedQueryPipeline fetches parent chunk when child matches and includes it in context (step 6.5, new `VectorStoreProvider.retrieve()`) +- [x] Parent expansion is configurable (`VEKTRA_PARENT_EXPANSION_ENABLED`, default off) +- [x] Token budget accounts for expanded parent chunk size (expansion runs before ARCH-055 allocation) +- [x] Tested: 17 unit tests (linkage, exclusion, expansion, budget); measured on `eval-full` — expansion is grounding-neutral there (35/55 with and without, zero per-question flips, avg sources 1.4 → 1.0 via sibling merge). The multi-chunk collapse it targeted turned out to be upstream: the rerank+threshold funnel wipes all candidates before expansion (TECH-007). The corpus also understates expansion benefit (short self-contained articles — TECH-005 collection 1 is the real test bench). + +**Resolution (2026-07-12, Sprint 3)**: shipped default-off on `feat/feat-017-parent-chunk-expansion`. Measured A/B on `eval-full` reingested dual (105 points = 22 parents + 83 children): dual chunking alone costs ~6.5pp retrieval hit vs fixed (children stop rolling overlap across 1500-token parent boundaries; IT-F-13, IT-R-02, EN-F-03 flip to miss). Full numbers in plan `20260712-sprint3-rag-quality` Notes. + +--- + +### TECH-005: Eval suite expansion (collections, casistiche, public regression slice) + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: Sprint 3 baseline eval (plan `20260712-sprint3-rag-quality`) - the excerpt corpus saturates hit rate; the full-corpus variant (`tests/eval/dataset-full.jsonl`) exposed multi-chunk collapse. Operator wants broader casistiche coverage for performance, regression and tuning decisions. + +**Context**: today the harness has two datasets over two corpora (excerpts 12 chunks, `eval-full` 78 chunks) plus a documented "dirty extraction" trap (senato.it combined PDF: pdfplumber fuses words, IT hit rate 72% vs 88% clean - see `tests/eval/README.md`). Each new collection must cover a failure mode the existing ones cannot express; ground truth is keyword-based (chunking-independent), generated by an LLM reading the FULL documents (not the chunks, to avoid single-chunk bias), with human spot-check on a sample and adversarial entries authored against corpus gaps. + +**Collections to add, in value order for the e-learning vertical**: +1. Long structured "textbook" document (chapters/sections, context-dependent definitions) - the real FEAT-017 test bench; constitution articles are short and self-contained, parent expansion benefit is understated there. +2. Multi-document corpus with thematic distractors (e.g. Costituzione + UDHR + ECHR in one namespace) - stresses reranker discrimination and per-document citation correctness (FEAT-021). +3. Multi-turn conversational scenarios - harness currently never sends `conversation_id`; needed for FEAT-018 verification anyway. +4. Tables/exact-value questions (dates, numbers, formulas) - discriminates BM25 vs dense and table extraction. +5. Cross-lingual as a formal dataset category (EN queries on IT corpus and vice versa). +6. Dirty-extraction suite (senato PDF) as a separate ingest-robustness track with its own expectations. + +**Public regression slice**: add a small external benchmark for component regressions (embedding/reranker swaps), separate from the domain datasets: SQuAD-it subset for Italian (paragraph containing the answer = expected passage) and optionally BEIR/SciFact for English. Note: no public dataset provides "expected reranker scores" - they provide relevance judgments; assert ranking metrics (nDCG/MRR), never absolute score values (RRF scores are 1.0/0.5/0.33 by construction). + +**Generative metrics**: TECH-002's RAGAS AC was never implemented (no `ground_truth_answer` in datasets, no judge). Decide whether to add an LLM-judge stage (local vLLM as judge) with ground-truth answers on a subset. + +**Acceptance criteria**: +- [ ] At least the textbook collection + multi-turn runner implemented with documented ground-truth methodology +- [ ] Public regression slice runnable via `make eval-retrieval EVAL_ARGS=...` with recorded baseline +- [ ] `tests/eval/README.md` updated with collection matrix and when to use which +- [ ] Decision recorded on RAGAS/LLM-judge (implement or drop the TECH-002 AC) + +--- + +### TECH-006: Extractor/OCR bake-off and per-document routing design + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: operator request (system instance runs `unstructured` for comparison vs pdfplumber); OCR landscape research 2026-07-12 (sources in the entry). + +**Context**: ingestion quality gates retrieval quality (see the senato.it PDF case: fused words invalidate keyword ground truth). Current extractors: pdfplumber (default) and unstructured `strategy="auto"` (tesseract OCR fallback; **image ships English tesseract only - no `tesseract-ocr-ita`**). The DocumentExtractor Protocol routes per MIME type only; the pdfplumber-vs-unstructured choice is global config (`_build_extractor_registry`, `vektra-ingest/pipeline.py:78-112`). + +**Research findings (July 2026)**: +- "GLM OCR" is real: Zhipu GLM-OCR, 0.9B, MIT weights, #1 OmniDocBench v1.5 (94.62), very fast (MTP), vLLM-servable; Italian NOT in the official language list - must be validated empirically. +- PaddleOCR-VL 1.6 (Apache-2.0): highest composite (96.33 OmniDocBench v1.6), explicit Italian support (111+ langs), official NVIDIA Blackwell setup guide, vLLM/SGLang backends. +- MinerU 3.4 (Apache-based license): end-to-end PDF->MD with built-in scanned-PDF auto-detection, reading order, cross-page tables, 109-lang OCR incl. Italian; `http-client` backend can point at a remote vLLM. +- Docling (IBM, MIT): framework with pluggable OCR engines, selective OCR of bitmap regions, `force_full_page_ocr`; cleanest MIT-licensed lib companion/replacement for unstructured. +- unstructured supports swapping its OCR engine via `OCR_AGENT` env (tesseract -> paddle) without code changes. +- Marker/Surya/Chandra excluded (GPL / OpenRAIL-M commercial restrictions); olmOCR-2 excluded (English-focused); GOT-OCR2.0/Nougat superseded. +- Zero-install baseline available today: the active vLLM model (qwen36-35b-a3b, vision-capable) can be prompted as a page-image extractor. + +**Proposed approach**: +1. Quick wins: add `tesseract-ocr-ita` (build arg for language packs) to the INSTALL_UNSTRUCTURED image; evaluate `OCR_AGENT=paddle`. +2. Bake-off on a real IT+EN course-material corpus (lecture PDFs, scanned handouts, slides): pdfplumber vs unstructured(auto) vs MinerU vs PaddleOCR-VL vs GLM-OCR (Italian validation), scored with olmOCR-bench-style per-page checks + the TECH-005 dirty-extraction suite. +3. Routing design (follow-up FEAT): per-page/per-document signals (embedded text layer, image-coverage ratio, garble score, language) choosing extractor; MinerU/Docling already embed such routing if delegating at document level. + +**Traceability**: ARCH-030 (pdfplumber extraction), ARCH-042 (extractor dispatch), ADR-0007, TECH-005 + +**Acceptance criteria**: +- [ ] Comparison report with per-tool scores on the shared corpus (IT + EN, native + scanned) +- [ ] Default extractor decision recorded (keep pdfplumber / switch / route) +- [ ] Routing feature filed as FEAT with concrete signals if the bake-off justifies it +- [ ] Image language packs fixed (tesseract-ita) regardless of outcome + +--- + +### TECH-007: Multi-part questions wiped by rerank+threshold funnel (multi-chunk collapse root cause) + +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-13 | **PR**: #92 +**Origin**: FEAT-017 measurement (plan `20260712-sprint3-rag-quality`) - expansion turned out to be downstream of the real failure. +**Evidence**: `vektra-internal/stack/20260713-tech007-retrieval-rescue.md` (+ per-question artifacts in `20260713-tech007-eval-artifacts/`) + +**Context**: on `eval-full`, 9/10 multi-chunk questions end with `retrieval_filter before=5 after=0` → `no_relevant_context` → refusal, despite 90% raw retrieval hit for the category. Cause: bge-reranker-v2-m3 scores each partial-answer chunk of a comparative/multi-part question low (each chunk answers only one part), and `VEKTRA_MIN_RELEVANCE_SCORE=0.15` — calibrated in the tuning sprint on single-fact questions (DEBT-010) — wipes the entire candidate set. Evidence (MC-01, eval mode traces): max reranker score 0.088 on a candidate whose raw RRF score was 0.61. Parent expansion (FEAT-017) never runs because zero results survive the filter. + +**Candidate directions** (evaluate, do not assume): (a) floor semantics - keep top-N post-rerank chunks regardless of threshold when the raw retrieval score was strong (e.g. min(top_k, after_rerank) >= 2); (b) per-category or per-score-source thresholds (reranker scores are not calibrated on the same scale as RRF); (c) query decomposition for multi-part questions (rewrite step already exists, ARCH-061); (d) rescore against the parent text instead of the child (combines with FEAT-017). + +**Resolution**: measured score distributions showed the collapse was wider than multi-chunk (also 4 reasoning + 2 factual wiped, hence grounded 35/55) and that no static threshold separates multi-chunk from adversarial (wiped MC max-rerank 0.005-0.088 vs wiped ADV 0.000-0.142, full overlap) — the filter cannot discriminate, and the reranker already passes 4-5/9 adversarial to the LLM today. Chose direction (a) in minimal form: **rescue only-when-empty** (`VEKTRA_RETRIEVAL_RESCUE_TOP_K`, default 0 = off; `VEKTRA_RETRIEVAL_RESCUE_FLOOR`, default 0.02): when the threshold empties the set, keep the top-N chunks above the floor and let strict grounding arbitrate. Discarded: (b) does not discriminate; (c) larger feature, downstream; (d) per-query cost, combinable later. Measured with `top_k=3, floor=0.005`: grounded 35/55 → 54/55, factual 19→21/21, reasoning 11→15/15, multi-chunk 1→10/10 by the harness metric — honestly: 2-3/10 substantially complete answers, 7 informed refusals that explain the gap (candidates for bi-document comparatives never include chunks of both documents: a candidate-coverage limit upstream of the filter, not a funnel issue). Adversarial: 0 answered-without-context, 0 hallucinations on manual review of all 9 (rescued ones give informed refusals or correct corrective answers). `top_k=5` control run equivalent within LLM variance. Latency unchanged. + +**Traceability**: ARCH-056 (retrieval quality controls), ADR-0021, DEBT-010, FEAT-017, TECH-005 + +**Acceptance criteria**: +- [x] Reproduce with the eval harness and document the score distributions per category +- [x] Chosen mitigation implemented behind config, default preserving current single-fact behavior +- [x] `eval-full` multi-chunk grounded moves from 0-1/10 without regressing factual (19/21) or adversarial refusals (no answered-without-context) +- [x] Decision and numbers recorded in the sprint plan and vektra-internal + +--- + +### FEAT-024: Remote embedding and reranker providers (TEI) + +**Status**: completed | **Priority**: medium | **Created**: 2026-07-12 | **Completed**: 2026-07-13 | **PR**: #93 +**Origin**: deployment modularity review 2026-07-12 - the host workstation already serves TEI instances (bge-m3, qwen3-embedding); Vektra cannot use them. +**Evidence**: `vektra-internal/stack/20260713-feat024-tei-providers.md` (+ artifacts in `20260713-feat024-eval-artifacts/`) + +**Context**: `VEKTRA_EMBEDDING_PROVIDER` documents a `tei` option (config.py:74) but **no TEI provider exists**: `main.py:129-137` unconditionally instantiates in-process `SentenceTransformersProvider`; the compose even ships a `tei` profile service nobody can talk to. The reranker likewise runs in-process only (`rerankers` lib; the `cohere` path never passes an api_key, so it is dead as wired - reranker.py:122). Consequences: every Vektra instance duplicates embedding/reranker compute in-container (CPU), and shared GPU/CPU inference services on the host cannot be reused. + +**Design**: +- `TEIEmbeddingProvider` implementing EmbeddingProvider over TEI HTTP (`POST /embed` or OpenAI-compatible `/v1/embeddings`), env: `VEKTRA_TEI_URL`, `VEKTRA_TEI_API_KEY`; `dimensions()` from TEI `GET /info`. Register on `VEKTRA_EMBEDDING_PROVIDER=tei`. +- `TEIRerankerProvider` over TEI `POST /rerank` (`{query, texts}` -> `[{index, score}]`, sigmoid scores; TEI serves bge-reranker-v2-m3 - one TEI instance per model). New `VEKTRA_RERANK_PROVIDER=tei` + endpoint/key env vars. Fix the cohere api_key pass-through in passing or remove the dead option. +- **Dimension plumbing**: `QdrantVectorStoreProvider` defaults `dense_dimensions=384` and main.py never passes it (qdrant.py:88, main.py:166-171) - a latent bug for any non-384 model; ensure collection creation uses the active provider's dimensions and document the reindex-on-model-change requirement (bge-m3 is 1024-dim). +- Note: TEI serves bge-m3 dense only (no sparse output for bge-m3); Vektra's BM25 sparse via fastembed stays client-side, hybrid keeps working. + +**Why it matters beyond dedup**: the current embedding model (paraphrase-multilingual-MiniLM-L12-v2) has **max_seq_length 128 tokens** - our 500-token chunks are silently truncated at embedding time (dense sees only the chunk head; BM25 sees the full text). bge-m3 (8192-token window, MIRACL dense nDCG@10 69.2 vs mE5-large 66.6; MiniLM sits 16-22 nDCG points below even mE5 on European-language retrieval per PL-MTEB) is the natural upgrade candidate, testable via TEI without fattening the container. + +**Resolution (2026-07-13)**: implemented as designed with two deltas: (1) TEI 1.9.3 `/info` does not expose the embedding size, so `dimensions()` probes `/embed` as fallback (both paths unit-tested); (2) found and fixed in passing a startup blocker: `check_embedding_model` resolved the provider by the hardcoded `sentence-transformers` name, so startup failed with any other provider (now uses the `default` alias). Measured on eval-full questions (same dual chunks, reindexed via a fresh `eval-tei` namespace): **bge-m3 via TEI retrieval hit 93.5% / MRR 0.8478 vs MiniLM dual 82.6% / 0.7029 (+10.9pp)** - beats even the fixed-chunking MiniLM baseline (89.1%/0.8062), confirming the 128-token truncation hypothesis; e2e grounded 54/55 stable, MC answers improve in substance (MC-02 produces a real bi-document comparison; kw 7/25 vs 4/25), p50 +0.7s (TEI on CPU). TEI reranker smoke: factual query scores 0.75 (2 survive the threshold), comparative query all-below-threshold rescued by TECH-007 (`rescued=3`) - full funnel verified with both remote providers, authenticated. Discovery filed under BUG-021: `run_reindex` stores through hardcoded pgvector, so reindex-into-Qdrant silently writes nothing (worked around via fresh-namespace ingest). Switching the default embedding to bge-m3 is a separate decision (needs full corpus re-ingest and a TECH-005-grade bench). + +**Traceability**: ADR-0013 (EmbeddingProvider Protocol), ARCH-035, ARCH-036, ADR-0021 + +**Acceptance criteria**: +- [x] `VEKTRA_EMBEDDING_PROVIDER=tei` works end-to-end (ingest + query) against a TEI instance with api key +- [x] Collection created with the provider's real dimensions; clear error on dimension mismatch with an existing collection (verified live: 384-vs-1024 startup warning with remediation) +- [x] `VEKTRA_RERANK_PROVIDER=tei` reranks via TEI /rerank with scores compatible with the threshold filter +- [x] Embedding-model comparison (MiniLM in-process vs bge-m3 via TEI) run with the TECH-005/existing harness and recorded +- [x] Docs: configuration.md + .env.example cover the new provider options + +--- + +### INFRA-007: Publish versioned container images on release (GHCR) + +**Status**: planned | **Priority**: medium | **Created**: 2026-07-12 +**Origin**: system-instance deployment 2026-07-12 - updates currently require a local `docker build` from a git checkout on every host. + +**Context**: no workflow publishes images (`release.yml` is a disabled placeholder, `if: false`, "Phase 1 releases are tagged manually"); the integration workflow builds only for its own tests. Deployments (e.g. the workstation rootful instance) must clone + build locally, which is slow and duplicates work per host. + +**Proposed approach**: GitHub Actions workflow on tag push (`v*`): build the image (both `INSTALL_UNSTRUCTURED=true` and `false` variants, e.g. tags `X.Y.Z` and `X.Y.Z-ocr`) and push to `ghcr.io/vektralabs/vektra`. Deployment update flow becomes `docker compose pull && docker compose up -d`. Consider enabling the semantic-release placeholder later; out of scope here. + +**Acceptance criteria**: +- [ ] Tag push publishes `ghcr.io/vektralabs/vektra:{version}` and `{version}-ocr` (multi-stage cache enabled) +- [ ] Image labels carry version + commit (OCI labels) +- [ ] README/deploy docs updated: pull-based deployment documented +- [ ] Existing tag flow unchanged (manual tagging still cuts the release) --- @@ -893,9 +1031,11 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou ### DEBT-025: Isolate unit tests from the developer's local .env -**Status**: planned | **Priority**: low | **Created**: 2026-07-12 +**Status**: completed | **Priority**: low | **Created**: 2026-07-12 | **Completed**: 2026-07-12 **Origin**: discovered during the DEBT-024 sweep (PR #80): `make test` fails locally with 4 errors while CI is green. +**Resolution**: new `vektra-shared/tests/conftest.py` autouse fixture scrubs `VEKTRA_*` (plus `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`) from `os.environ` per-test via monkeypatch. Root cause confirmed: importing litellm during collection runs `dotenv.load_dotenv()`, leaking the repo `.env` into the process environment — running `vektra-shared/tests/test_config.py` alone passes, collecting it together with any litellm-importing package reproduces the 4 failures. Production settings loading untouched (no settings class uses `env_file`). + **Context**: 4 tests in `vektra-shared/tests/test_config.py` (`TestLLMConfig::test_defaults`, `TestQueryPipelineConfig::test_eval_mode_default_false`, `test_debug_log_queries_default_false`, `TestVektraSettings::test_defaults_with_required_only`) assert configuration defaults, but when the full suite runs from the workspace root the developer's `.env` leaks into `os.environ` (something imported during collection loads dotenv, e.g. litellm), so machine-specific values (eval_mode=true, custom port, LLM keys) override the defaults and the assertions fail. CI never sees this because runners have no `.env`. Current workaround: temporarily move `.env` away before `make test`. **Proposed approach**: a `vektra-shared/tests` (or workspace-level) autouse fixture that snapshots and scrubs `VEKTRA_*` variables from `os.environ` for the default-assertion tests, or `monkeypatch.delenv` on the specific vars. Alternatively point pydantic-settings at a nonexistent env file in tests via `_env_file=None`. @@ -907,6 +1047,24 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou --- +### BUG-022: INSTALL_UNSTRUCTURED image build broken — torchvision resolved from PyPI against torch+cpu + +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-12 +**Origin**: first production build with `INSTALL_UNSTRUCTURED=true` (rootful system-instance deployment on the dev workstation, 2026-07-12). + +**Context**: the OCR image variant has never built successfully. `uv sync --extra ocr` resolves `torchvision` — a transitive dependency via `unstructured[pdf]` → unstructured-inference → timm — from PyPI, whose wheels are compiled against CUDA torch, while `torch` itself is pinned to the `pytorch-cpu` index (vektra-index `[tool.uv.sources]`). At image build the model pre-cache step (`Dockerfile:126`) fails importing sentence_transformers with `RuntimeError: operator torchvision::nms does not exist`. CI never builds with the flag, so the breakage stayed invisible since the extra was introduced. + +**Resolution**: declare `torchvision>=0.25,<0.26` in the `ocr` extra with `[tool.uv.sources] torchvision = { index = "pytorch-cpu" }` in `vektra-ingest/pyproject.toml` (same pattern as torch in vektra-index); relock — torchvision flips to `0.25.0+cpu` from the CPU index, torch stays at 2.10.0 (upper bound `<0.26` keeps the relock surgical). New path-filtered workflow `.github/workflows/docker-ocr-build.yml` builds the `INSTALL_UNSTRUCTURED=true` image whenever Dockerfile/uv.lock/ingest deps change, so the variant cannot silently regress again. + +**Note**: the OCR image ships English tesseract only (`tesseract-ocr-eng`); the missing Italian language pack is tracked as a TECH-006 quick win, out of scope here. + +**Acceptance criteria**: +- [ ] `docker build --build-arg INSTALL_UNSTRUCTURED=true .` succeeds from a clean cache +- [ ] `uv.lock` resolves torchvision from the pytorch-cpu registry +- [ ] CI builds the OCR variant on changes to Dockerfile / uv.lock / ingest deps + +--- + ### DEBT-026: Tune Prometheus instrumentation exclusions (health/docs endpoints) **Status**: planned | **Priority**: low | **Created**: 2026-07-12 @@ -923,6 +1081,27 @@ Three near-duplicates is the threshold where extraction starts to pay off (a fou --- +### BUG-021: /api/v1/search hardwired to pgvector — empty results and no hybrid in Qdrant mode + +**Status**: completed | **Priority**: high | **Created**: 2026-07-12 | **Completed**: 2026-07-12 +**Origin**: Sprint 3 baseline eval (plan `20260712-sprint3-rag-quality`): `make eval-retrieval` returned zero results for all 55 questions against a healthy stack. + +**Context**: the search endpoint (`vektra-index/api.py`) instantiated `PgvectorProvider` directly and looked up the sparse provider in `request.app.state.sparse_embedding_provider`. Neither matches the app wiring: `main.py` registers providers in the ProviderRegistry (`vector_store`/`default` is overridden by Qdrant when `VEKTRA_VECTOR_STORE_PROVIDER=qdrant`; sparse under `sparse_embedding`/`default`; nothing is ever set on `app.state.sparse_embedding_provider`). Consequences in Qdrant deployments: (1) every search ran against the empty Postgres `document_chunks` table and returned `{"results": [], "total": 0}` with HTTP 200; (2) hybrid mode always fell back to dense with a `sparse_embedding_not_registered` warning even though FastEmbedBM25 was registered at startup. The RAG pipeline (`/api/v1/query`) was unaffected — it resolves providers from the registry — which is why the bug stayed invisible until the retrieval eval ran in qdrant mode. + +**Resolution**: the endpoint now resolves embedding, sparse embedding, and vector store from `request.app.state.registry` (same contract as the pipeline). The per-request `SentenceTransformersProvider` instantiation and the now-unused `session` dependency were removed. Unit tests added (`vektra-index/tests/test_api_search.py`): registry resolution, hybrid→dense fallback without sparse, hybrid with sparse. + +**Same family, not fixed here**: `POST /documents/{id}/chunks`, `DELETE /documents/{id}` and `GET /stats` still hardcode pgvector (the stats-vs-Qdrant mismatch was already a known issue). Also `run_reindex` (`vektra-index/reindex.py`): it re-embeds with the registry's active embedding provider but stores through a hardcoded `PgvectorProvider`, so in Qdrant mode a reindex reports "completed" while the Qdrant collection receives nothing (found during the FEAT-024 live smoke, 2026-07-13: reindexing eval-full to v2 wrote to Postgres only). Track separately if needed. + +**Traceability**: ARCH-039 (ProviderRegistry), ARCH-051 (full-store contract), TECH-002 (eval harness) + +**Acceptance criteria**: +- [ ] `/api/v1/search` returns results in Qdrant mode (dense and hybrid) +- [ ] Hybrid uses the registered sparse provider (no spurious fallback) +- [ ] Unit tests pin registry-based provider resolution +- [ ] `make eval-retrieval` produces non-zero hit rate against the eval corpus + +--- + ### INFRA-005: Docker log persistence across container restarts **Status**: planned | **Priority**: medium | **Created**: 2026-03-23 diff --git a/.s2s/plans/20260712-sprint3-rag-quality.md b/.s2s/plans/20260712-sprint3-rag-quality.md new file mode 100644 index 00000000..938f9de4 --- /dev/null +++ b/.s2s/plans/20260712-sprint3-rag-quality.md @@ -0,0 +1,347 @@ +# Implementation Plan: Sprint 3 — RAG quality + +**ID**: 20260712-sprint3-rag-quality +**Status**: active +**Branch**: one branch/PR per item (`chore/debt-025-test-env-isolation`, `feat/feat-017-parent-chunk-expansion`, `feat/feat-021-namespace-citations`, FEAT-018 branch only if verification justifies it) +**Created**: 2026-07-12T08:39:46Z +**Updated**: 2026-07-12T09:35:00Z + +## Traceability + +**Source**: DEBT-025, FEAT-017, FEAT-018 (conditional on verification), FEAT-021 +**Source Type**: backlog + +Roadmap note: Sprint 3 in `vektra-internal/stack/20260321-implementation-roadmap-post-phase2.md` +listed FEAT-008, FEAT-013, FEAT-004, OQ-014. Deltas: FEAT-004 shipped in v0.5.0; +FEAT-013 is subsumed by FEAT-021 (citation titles/snippets); FEAT-008 stays out of +scope until it has a design (`/s2s:design`); OQ-014 not in this sprint. + +## References + +### Requirements +- REQ-055: response and citation traceability @.s2s/requirements.md (FEAT-021) +- REQ-051: no user text in traces (constrains multi-turn verification tooling; eval mode only) + +### Architecture +- ARCH-037: ChunkingStrategy (dual parent/child) @.s2s/architecture.md +- ARCH-055: token budget allocation @.s2s/architecture.md +- ARCH-056: retrieval quality controls @.s2s/architecture.md +- ARCH-054: composable Jinja2 templates @.s2s/architecture.md +- ARCH-047: namespace as first-class entity with metadata @.s2s/architecture.md +- ARCH-050: three-tier evaluation strategy @.s2s/architecture.md + +### Decisions +- ADR-0019: RAG evaluation strategy @.s2s/decisions/ADR-0019-rag-evaluation-strategy.md +- ADR-0021: retrieval quality controls @.s2s/decisions/ADR-0021-retrieval-quality-controls.md +- ADR-0023: conversational query rewriting @.s2s/decisions/ADR-0023-conversational-query-rewriting.md +- ADR-0020: prompt template architecture @.s2s/decisions/ADR-0020-prompt-template-architecture.md +- ADR-0025: learn chatbot widget @.s2s/decisions/ADR-0025-learn-chatbot-widget.md + +### Dependencies +- none (TECH-002 eval harness merged in 20260325-rag-retrieval-quality) + +## Overview + +Sprint objective: improve RAG answer quality **and measure it**. Every quality change +is bracketed by eval-harness runs (TECH-002) on the same dataset and corpus config +(Combo D: multilingual MiniLM embeddings, chunk 500/100, hybrid BM25+dense, reranker +bge-reranker-v2-m3 top_k=5, threshold 0.15, advanced pipeline). + +Order: (1) DEBT-025 warm-up so `make test` is green on dev machines, (2) baseline +eval, (3) FEAT-017 parent chunk expansion measured before/after, (4) FEAT-018 +verification-first — implement only if multi-turn tests show FEAT-020 did not +already mitigate it, (5) FEAT-021 per-namespace citations. + +## Design Notes + +Ground truths from codebase analysis (2026-07-12, develop @ ca25717) that correct +the backlog assumptions: + +- **Parent-child linkage is NOT stored** (FEAT-017 premise in backlog is wrong). + `DualStrategyChunking` builds the hierarchy in memory (`vektra-ingest/chunking.py:259-313`) + but `run_ingest` drops `chunk.parent_id` when building `ChunkEmbedding` + (`vektra-ingest/pipeline.py:400-418`); `ChunkEmbedding` has no parent_id field + (`vektra-shared/types.py:115-123`); Qdrant payload carries no parent_id + (`vektra-index/providers/qdrant.py:181-187`); `DocumentChunkOrm.parent_id` is always + NULL. Only `metadata.chunk_level` (parent|child) survives. FEAT-017 must plumb the + id through and **reingest** with `VEKTRA_CHUNKING_STRATEGY=dual`. +- **Parents ARE indexed and searchable today** when dual is enabled: `run_ingest` + embeds and upserts all chunks with no level filter; neither Qdrant `_build_filter` + nor pgvector excludes `chunk_level=parent`. The "search excludes parents" AC is + a real behavior change. +- **Qdrant mode keeps chunk text only in Qdrant**: with `VEKTRA_VECTOR_STORE_PROVIDER=qdrant` + (local setup), `document_chunks` in Postgres is empty; parent text must be fetched + via the vector store (Qdrant retrieve-by-id), not SQL. +- Parent size is hardcoded `chunk_size*3` at `vektra-ingest/pipeline.py:343-348` + (backlog's "3000 tokens" only holds for chunk_size=1000; with Combo D 500 it is 1500). +- **Qdrant filter builder is must-only** (`qdrant.py:376-431`); FEAT-018 chunk + exclusion needs `must_not` support. pgvector accepts `raw_filters` but silently + drops them (`pgvector.py:100-130`) — paper escape hatch. +- **Per-turn chunk ids are recoverable**: `conversation_turns.response_id` is populated + (BUG-013/DEBT-011) and `query_traces.chunks_retrieved` stores `[{chunk_id, score}]` + keyed by response_id — but only when trace storage is on (auto: dev on / prod off). +- **FEAT-020 pattern to clone for FEAT-021**: `resolve_grounding_mode` / + `resolve_show_sources` in `vektra-shared/namespace.py:19-82` read `namespaces.config` + JSONB; the API layer resolves and passes the value on `QueryRequest` + (`vektra-core/api.py:263-280`). Add `resolve_citations_enabled` the same way; + admin `PATCH /admin/namespaces/{id}/config` whitelist must accept the new key. +- **context.j2 uses ``**, not the `` shown + in the FEAT-021 backlog entry. `SearchResult.metadata` already carries `source_file` + and `page` from the Qdrant payload; `document_name` comes from `_fetch_document_names` + (currently applied to sources only, not to context rendering). Template edits change + `prompt_version` (trace comparability across the sprint). +- **Widget has no citation rendering**: `renderMarkdown` (bold/italic/code/links/lists + only) escapes literal `[1]`; inline citation markers hook into `renderInline` + (`vektra-learn/widget/src/markdown.js:136-153`). Bundle via esbuild to + `vektra-learn/static/vektra-chat.js`. +- **Eval harness**: HTTP clients only (`VEKTRA_API_URL` + `VEKTRA_API_KEY`; `.api-key` + file no longer exists locally — use `.env` value). e2e computes answerability + + latency; **RAGAS is not implemented** despite the TECH-002 AC text, and the dataset + has no `ground_truth_answer` fields. Retrieval metrics: hit rate, MRR, precision@k. + Dataset: 55 questions, all `namespace=default` (21 factual / 15 reasoning / + 10 multi-chunk / 9 adversarial; 37 IT / 18 EN). No multi-turn entries and no + `conversation_id` support in the scripts: FEAT-018 verification needs a small + multi-turn runner. +- **Local corpus state (2026-07-12)**: Qdrant collection `vektra` has 656 points; + the 12 in namespace `default` are the intact excerpt eval corpus from 20260325 + (`costituzione_italiana.md` v2 + `udhr_excerpts.md` v2 + `sample.pdf`). Current + local chunking is `fixed` 500/100. The full Costituzione PDF (97 chunks) was + deliberately removed from `default` back then; FEAT-017 will reintroduce a + larger corpus with `dual` chunking. + +## Tasks + +### 1. DEBT-025 — isolate unit tests from local .env (branch `chore/debt-025-test-env-isolation`) +- [x] Root cause confirmed: litellm import-time `load_dotenv()` during collection +- [x] Autouse fixture in `vektra-shared/tests/conftest.py` scrubs `VEKTRA_*` + external keys +- [x] `make test` green with populated `.env` (638 passed) + `make lint` green +- [x] Backlog entry updated (completed + resolution), changelog entry +- [x] PR created and merged (#84; #85 BUG-021, #86 backlog, #87 BUG-022 merged the same day) + +### 2. Baseline eval (no branch; results recorded, not committed as code) +- [x] Eval corpus verified intact in namespace `default` (excerpt corpus, 12 chunks — no reingest needed) +- [x] BUG-021 discovered and fixed (search endpoint hardwired to pgvector — see Notes; branch `fix/bug-021-search-registry-providers`) +- [x] `make eval-retrieval` baseline recorded (hit rate, MRR, precision@k, per-category) +- [x] `make eval-e2e` baseline recorded (grounded rate, no-ctx, latency p50/p95) +- [x] Numbers logged in this plan (Notes) and in vektra-internal (`stack/20260712-sprint3-baseline-eval.md`) + +### 3. FEAT-017 — parent chunk expansion (branch `feat/feat-017-parent-chunk-expansion`) +- [x] Propagate `parent_id` into `ChunkEmbedding` → Qdrant payload + pgvector column; + keep deterministic child/parent ids at store time (pgvector now honors + caller-provided uuid5 ids instead of generating uuid4) +- [x] Search excludes `chunk_level=parent` by default (both providers) +- [x] `VEKTRA_PARENT_EXPANSION_ENABLED` (default false) in `QueryPipelineConfig` + + mirrored in `VektraSettings` +- [x] Expansion step in AdvancedQueryPipeline: fetch parent text by id via new + `VectorStoreProvider.retrieve()`, replace child text **before** token + budgeting (ARCH-055); children of the same parent collapse into the + highest-scored one (runs after the 80% overlap dedup, so no interaction) +- [x] Trace metadata records expansion (children_expanded, siblings_merged, + parents_fetched) +- [x] Reingest eval corpus with `dual` strategy; measured both (see Notes): + A dual-no-expansion ≈ baseline on e2e but -6.5pp retrieval hit (boundary + effect); B expansion-on: zero e2e flips, sources 1.4 → 1.0 +- [x] Unit tests: store-time linkage, search filter, expansion logic, budget + accounting (17 new tests; suite 658 passed) + +### 4. FEAT-018 — verification first (no branch unless justified) +- [x] Multi-turn scenario runner against `/api/v1/query` with `conversation_id` + (same-topic follow-up, topic switch, "give me others", negation) +- [x] Evaluate with FEAT-020 grounding modes: rewrite + history already mitigate + topic switch and negation fully (0 shared chunks, high rerank scores); + "give me more" is the one genuine case (3/3 identical chunks, graceful + degradation via history); same-topic follow-up dies at the threshold + (TECH-007), which exclusion would worsen +- [x] Decision recorded in backlog: **no-go for Sprint 3, deferred behind + TECH-007** — evidence in the FEAT-018 backlog entry and Notes below + +### 5. FEAT-021 — per-namespace citations (branch `feat/feat-021-namespace-citations`) +- [x] `resolve_citations_enabled` in `vektra_shared/namespace.py` (clone of FEAT-020 + pattern), default false; admin config PATCH whitelist + GET resolved defaults + extended +- [x] `QueryRequest.citations_enabled` resolved at API layer (core + learn), passed + to renderer +- [x] `system.j2` Rule 1 conditional (cite `[id]` inline when enabled and context + present) +- [x] `context.j2`: `title` attribute (document_name + page, composed at + prompt-build time) when enabled; template tolerant of chunks without title +- [x] `SourceRef.title` in API responses (core sync + streaming, learn course + response); `source_file`/`page` were already in `SearchResult.metadata` +- [x] Widget: `[n]` markers rendered as superscripts with source-title tooltip + wired in `addSources()`; bundle built by the Docker widget-builder stage + (static/ is gitignored) +- [x] Default-off behavior byte-identical prompts, asserted in template tests + (prompt_version changes because template files changed — documented in + changelog) +- [x] Unit tests: namespace resolution (4), templates (6), pipeline flow (2), + admin integration (2 + resolved defaults); suite 672 passed + +## State & Data Lifecycle + +| State element | Created by | Updated by | Invalidated/Deleted by | +|---------------|-----------|------------|------------------------| +| Qdrant payload `parent_id` + `chunk_level` | ingest (FEAT-017) | reingest of document | document delete / collection reindex | +| `namespaces.config.citations_enabled` | admin config PATCH | admin config PATCH | key removal via PATCH null / namespace delete | +| `tests/eval/results_*.jsonl` | eval runs | overwritten per run | git history keeps baselines | +| conversation→chunk linkage (traces) | query pipeline (existing) | n/a (append-only) | analytics retention job | + +## Acceptance Criteria + +- [ ] Per-item ACs from `.s2s/BACKLOG.md` (DEBT-025 ✓, FEAT-017, FEAT-018-verification, FEAT-021) +- [ ] Baseline and post-change eval runs use the same dataset, corpus and config, + differing only in the variable under test; runs with per-question errors are + not accepted as baselines +- [ ] FEAT-018 go/no-go decision documented with test evidence in the backlog +- [ ] FEAT-021 default-off leaves current behavior unchanged (templates render + identically when disabled) +- [ ] `make lint` + `make test` green before every push + +## Testing Approach + +Unit tests accompany every code change (chunk linkage, search filters, expansion, +template conditionals, namespace resolution). Quality is measured with the TECH-002 +harness before/after each RAG-affecting change. Widget changes verified by rebuilding +the bundle and a manual smoke in the Moodle dev stack. + +### Test Infrastructure + +- **Required**: running stack (vektra + postgres + qdrant profile), eval corpus in + namespace `default`, reachable LLM (local vLLM via Tailscale IP), `VEKTRA_API_URL` + + `VEKTRA_API_KEY` env for the harness +- **Provided by**: `docker compose --profile qdrant up -d`; corpus reingest task in + items 2-3; existing `.env` (Combo D) +- **False-pass guard**: eval scripts mark per-question errors and print error counts; + a baseline with errors > 0 is rerun, not recorded. FEAT-018 verification failures + block implementation, not silently skip it. + +## Integration Notes + +- FEAT-021 touches the widget served by vektra-stack (`/static/learn/vektra-chat.js`); + the Moodle plugin embeds it unchanged — no vektra-moodle release needed unless new + `data-*` attributes are added (not planned: citations are namespace-driven). +- FEAT-017 requires a corpus reingest wherever dual strategy is enabled; local only, + no production deployments exist (no backward-compat constraints). +- Backlog drift found during analysis (FEAT-017 stored-linkage premise, FEAT-021 + template quote, TECH-002 RAGAS AC, DEBT-025 `_env_file` suggestion) is corrected + in the backlog as each item lands. + +## Notes + +- 2026-07-12: DEBT-025 completed on `chore/debt-025-test-env-isolation` + (fixture + docs, suite 638 passed / 3 skipped). Plan file rides the same PR. +- 2026-07-12: local stack recovered for eval: Qdrant container was down since ~May, + restarted from compose profile with existing volume (collection green, 656 points). + The 12 points in namespace `default` turned out to be the intact excerpt eval + corpus (`costituzione_italiana.md` v2, 6 chunks + `udhr_excerpts.md` v2, 4 + + `sample.pdf`, 2) — no reingest needed for the baseline. +- 2026-07-12: baseline eval was blocked twice, both fixed: (a) **BUG-021** — + `/api/v1/search` was hardwired to pgvector and read the sparse provider from a + never-populated `app.state` attribute → zero results for all 55 questions in + qdrant mode, hybrid always degraded to dense. Fixed on + `fix/bug-021-search-registry-providers` (registry-based resolution + endpoint + tests). (b) stale vLLM model id in local `.env` (`qwen35-27b-fp8` after a + `vllm-switch`; server serves `qwen36-35b-a3b-fp8`) → every LLM call failed with + NotFoundError and the pipeline returned context-only answers (answer null). + `.env` updated, container recreated with the BUG-021 fix baked in. +- 2026-07-12: **baseline recorded** (excerpt corpus, Combo D, qwen36-35b-a3b-fp8, + grounding strict; full report in vektra-internal + `stack/20260712-sprint3-baseline-eval.md`): + - retrieval (`/api/v1/search` hybrid, top_k=5): hit rate 100% (46 scored), + MRR 0.8957, precision@5 0.3261; MRR factual 0.8810 / multi-chunk 0.8200 / + reasoning 0.9667; EN 0.8595 / IT 0.9115; RRF score p50 0.50. Hit rate + saturates on the tiny corpus — MRR/precision are the sensitive metrics. + - e2e (`/api/v1/query`, 55 questions, 0 errors, 178s): grounded 38/55 (69%), + no_context 17/55, answered-without-context 0, avg sources 1.3, latency + p50 3069ms / p95 4371ms. By category: factual 19/21, reasoning 12/15, + **multi-chunk 3/10**, adversarial 4/9 (adversarial no_ctx is largely the + desired refusal). + - reading: multi-chunk is the bottleneck — 7/10 end in no_context despite 100% + retrieval hit: chunks are found by search but cut by rerank+threshold before + the prompt (avg sources 1.3). Primary target for FEAT-017. Consider + reingesting the full Costituzione PDF (97 chunks) for a more discriminative + FEAT-017 comparison. + - March numbers (factual 90 / reasoning 80 / multi-chunk 10) are not comparable: + different pipeline (pre BUG-015/016/017, pre FEAT-020) and per-question + results were never versioned (gitignored file, since overwritten). +- 2026-07-12: **FEAT-017 measured** (full corpus `eval-full`, Combo D, same + dataset/config as baseline; corpus reingested with `dual` 500/100, parent 1500: + 105 points = 22 parents + 83 children, parents excluded from search). + Baseline to compare (fixed 500/100, 78 chunks): retrieval hit 89.1% / + MRR 0.8062 / P@5 0.3174; e2e grounded 35/55, multi-chunk 0/10, avg sources 1.4. + - **Measure A** (dual + parent exclusion, expansion off): retrieval hit 82.6% / + MRR 0.7029 / P@5 0.3130; e2e grounded 35/55 (64%), multi-chunk 1/10, avg + sources 1.4, p50 3935ms. Dual chunking itself costs ~6.5pp hit rate on this + corpus: children no longer roll overlap across parent boundaries, and 3 + questions whose keywords straddle a 1500-token section edge flip to miss + (IT-F-13, IT-R-02, EN-F-03). + - **Measure B** (expansion on): e2e grounded 35/55, multi-chunk 1/10, avg + sources 1.0 (sibling merge working: same grounding with a more compact + prompt), p50 3813ms. Zero per-question flips vs A. Expansion verified live + in traces (`parent_expansion` step, children_expanded/parents_fetched). + - **Root cause of the multi-chunk collapse found (and it is upstream of + FEAT-017)**: 9/10 multi-chunk questions end with `retrieval_filter + before=5 after=0` — bge-reranker-v2-m3 scores each partial-answer chunk + of a comparative/multi-part question below `min_relevance_score` 0.15 + (MC-01 with eval mode: max rerank score 0.088 while the raw RRF candidate + was 0.61). The whole candidate set is wiped before expansion can run. + Filed as TECH-007. Parent expansion works as designed but cannot touch + this failure mode; the Costituzione corpus also understates its benefit + (short self-contained articles — see TECH-005 collection 1). + - Local dev `.env` now: `VEKTRA_CHUNKING_STRATEGY=dual`, + `VEKTRA_PARENT_CHILD_LEVELS=1`, `VEKTRA_PARENT_EXPANSION_ENABLED=true`, + `VEKTRA_EVAL_MODE=true` (left on for FEAT-018 trace inspection). +- 2026-07-12: **FEAT-018 verified — no-go, deferred behind TECH-007.** Four + multi-turn scenarios against `/api/v1/query` with `conversation_id` + (eval-full corpus, expansion on, eval-mode traces): topic switch and negation + fully mitigated by rewrite (ARCH-061) + history (FEAT-020) — chunks change + completely, rerank 0.99/0.82; same-topic follow-up ("quali limiti prevede + l'art. 21?") rewrites correctly but dies at the retrieval filter (max rerank + 0.093 < 0.15 — same TECH-007 signature as multi-chunk); "give me more" + confirms the FEAT-018 premise (3/3 identical chunks despite an explicit + "diversi da quelli già citati" rewrite) but degrades gracefully via history. + Chunk exclusion today would raise refusals on follow-ups; revisit after + TECH-007. Full per-turn evidence in the FEAT-018 backlog entry. +- 2026-07-13: **TECH-007 resolved (post-sprint, PR #92)** — retrieval-filter + rescue, `feat/tech-007-retrieval-rescue`. Reproduction with eval-mode traces + on all 55 questions showed (1) the collapse was wider than multi-chunk: + 4 reasoning + 2 factual also wiped (that is why grounded was 35/55); + (2) no static threshold separates multi-chunk from adversarial (wiped MC + max-rerank 0.005-0.088 vs wiped ADV 0.000-0.142, full overlap) — the filter + cannot discriminate, and the reranker already passes 4-5/9 adversarial to + the LLM (0.36-0.84) whose strict grounding answers correctively. Mitigation: + rescue only-when-empty behind `VEKTRA_RETRIEVAL_RESCUE_TOP_K` (default 0 = + off) + `VEKTRA_RETRIEVAL_RESCUE_FLOOR` (default 0.02); `retrieval_filter` + trace step gains `rescued`. Measured with top_k=3/floor=0.005 (same corpus, + expansion on): grounded 35/55 -> **54/55**, factual 19 -> 21/21 (kw 45/50), + reasoning 11 -> 15/15 (kw 41/46), multi-chunk 1 -> 10/10 by harness metric + (honest reading: 2-3/10 substantially complete, 7 informed refusals that + correctly explain the missing half — bi-document comparatives never get + chunks of both documents among the 20 candidates: candidate-coverage limit, + upstream of the filter). Adversarial: 0 answered-without-context, 0 + hallucinations (manual review 9/9). top_k=5 control run equivalent within + LLM variance; p50 unchanged (~3.9s). Full report: + `vektra-internal/stack/20260713-tech007-retrieval-rescue.md`; per-question + artifacts in `20260713-tech007-eval-artifacts/`. Dev `.env` adds + `VEKTRA_RETRIEVAL_RESCUE_TOP_K=3`, `VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.005`. +- 2026-07-13: **FEAT-024 implemented and measured (post-sprint, PR #93)** — + TEI remote providers, `feat/feat-024-tei-providers`. TEIEmbeddingProvider + (native /embed, /info dimensions with /embed probe fallback — TEI 1.9.3 + does not expose the size in /info), TEIRerankerService (/rerank, same + RerankResult semantics), Qdrant collection sized from the active provider + (fixes the hardcoded-384 latent bug; live 384-vs-1024 mismatch produces a + clear startup warning), cohere api_key pass-through fixed, startup warmup + fixed to resolve the `default` embedding alias (was hardcoded to + 'sentence-transformers' and broke any alternative provider). + **Comparison on eval-full questions (same dual chunks, ns eval-tei, + bge-m3 float32 CPU via TEI): retrieval hit 82.6% -> 93.5%, MRR 0.7029 -> + 0.8478 — beats even the fixed-chunking MiniLM baseline (89.1%/0.8062), + confirming MiniLM's 128-token truncation as a real retrieval cap.** E2e + grounded 54/55 stable, multi-chunk substance improves (MC-02 real + bi-document comparison, kw 7/25 vs 4/25), p50 +0.7s (CPU TEI). Reranker + smoke: factual 0.75 survives threshold, comparative all-below rescued by + TECH-007 (rescued=3) — full funnel verified with both remote providers. + Found in passing (filed under BUG-021 family): run_reindex stores through + hardcoded pgvector — reindex-into-Qdrant silently no-ops. Dev stack + restored to reference config (MiniLM + rescue); eval-tei namespace and + vektra-tei collection (105 points, 1024-dim) left in place for follow-ups; + bge-reranker TEI cache kept at /mnt/scratch/tei-rerank-cache. diff --git a/CHANGELOG.md b/CHANGELOG.md index 22332dfb..1122283a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,28 @@ Convention (Keep a Changelog 1.1.0): +## [0.6.0] - 2026-07-13 + +RAG quality release: parent chunk expansion, retrieval-filter rescue, per-namespace citations, remote TEI providers. + +### Added + +- **rag**: remote embedding and reranking via HuggingFace Text Embeddings Inference (FEAT-024). `VEKTRA_EMBEDDING_PROVIDER=tei` embeds through a TEI instance (`VEKTRA_TEI_URL`/`VEKTRA_TEI_API_KEY`, native `/embed` API) instead of in-process sentence-transformers, enabling shared host inference and long-window models (bge-m3: 8192 tokens vs MiniLM's 128, which silently truncates 500-token chunks today). `VEKTRA_RERANK_PROVIDER=tei` reranks through TEI `/rerank` (`VEKTRA_RERANK_TEI_URL`/`VEKTRA_RERANK_TEI_API_KEY`). The Qdrant collection is now sized from the active embedding provider's dimensions instead of a hardcoded 384 (latent bug for any non-384 model), with a clear startup error on dimension mismatch against an existing collection. The `cohere` rerank option now actually passes its API key (`VEKTRA_RERANK_API_KEY`); it was dead as wired. +- **rag**: optional retrieval-filter rescue for multi-part questions (TECH-007, `VEKTRA_RETRIEVAL_RESCUE_TOP_K` + `VEKTRA_RETRIEVAL_RESCUE_FLOOR`, default off). When `VEKTRA_MIN_RELEVANCE_SCORE` empties the candidate set, keep the top-N chunks above an absolute floor instead of refusing: the cross-encoder scores each partial-answer chunk of a comparative/multi-part question below the threshold (it answers only one part), so on the eval corpus 9/10 multi-chunk questions died at the filter with `before=5 after=0` despite 90% raw retrieval hit. With the rescue, borderline sets reach the LLM, which arbitrates via strict grounding. The `retrieval_filter` trace step now records a `rescued` count. +- **rag**: optional per-namespace inline source citations (FEAT-021, `citations_enabled` in the namespace config JSONB via `PATCH /api/v1/admin/namespaces/{id}/config`, default off, advanced pipeline only). When enabled, the system prompt instructs the LLM to add inline `[n]` markers matching the `` elements, the context template carries a `title` attribute ("filename, p.N"), and each returned source includes a `title` field; the learn widget renders the markers as superscripts with a tooltip. Default-off renders byte-identical prompts; `prompt_version` changes anyway because the template files changed (trace comparability note). +- **rag**: optional parent chunk expansion in the advanced query pipeline (FEAT-017, `VEKTRA_PARENT_EXPANSION_ENABLED`, default off). With `VEKTRA_CHUNKING_STRATEGY=dual`, retrieved child chunks are replaced with their parent chunk's text after the retrieval filter and before token budgeting; children of the same parent collapse into the highest-scored one. Parent-child linkage is now actually persisted (deterministic `uuid5(doc_id, position)` ids, `parent_id` in the Qdrant payload and in the pgvector column), a new `VectorStoreProvider.retrieve()` fetches chunks by id, and the trace records `children_expanded`/`siblings_merged`/`parents_fetched` in a `parent_expansion` step. + +### Changed + +- **index**: vector search now excludes parent-level chunks (`chunk_level=parent`) in both providers; parents are context material fetched by id during expansion, not retrieval targets. Only affects documents ingested with `dual` chunking, whose parents previously polluted search results. +- **index**: pgvector `store()` honors caller-provided UUID chunk ids (deterministic ids from ingest) instead of always generating random ones; non-UUID ids still fall back to random. + +### Fixed + +- **docker**: the `INSTALL_UNSTRUCTURED=true` image variant builds again (BUG-022). torchvision (transitive via unstructured-inference) resolved from PyPI with CUDA-built wheels while torch is pinned to the CPU index, crashing the build with `operator torchvision::nms does not exist`. It is now declared in the `ocr` extra and pinned to the pytorch-cpu index; a new path-filtered CI workflow builds the OCR variant so it cannot silently regress. +- **index**: `/api/v1/search` now resolves the embedding, sparse-embedding, and vector-store providers from the ProviderRegistry instead of hardcoding pgvector and reading a never-populated `app.state` attribute (BUG-021). In Qdrant deployments the endpoint returned zero results (it searched the empty `document_chunks` table) and hybrid mode always fell back to dense; the RAG pipeline (`/api/v1/query`) was unaffected. Found by the Sprint 3 baseline `make eval-retrieval` run. +- **tests**: unit tests are now hermetic against the developer's local `.env` (DEBT-025). Importing litellm during pytest collection loads `.env` into the process environment, which made 4 default-assertion tests in `vektra-shared` fail on dev machines while CI stayed green. An autouse fixture in `vektra-shared/tests/conftest.py` scrubs ambient `VEKTRA_*` variables; production settings loading is unchanged. The fixture is replicated in `vektra-core` and `vektra-ingest` (FEAT-017 surfaced the same leak there: ambient `VEKTRA_CHUNKING_STRATEGY=dual` broke 12 chunker and pipeline-default tests). + ## [0.5.1] - 2026-07-12 Security hardening: DEBT-024 dependency sweep, workflow permissions, admin login hardening. diff --git a/docs/reference/api.md b/docs/reference/api.md index 99dd9ba5..d1d75611 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -194,6 +194,7 @@ Request body (flat dict, one entry per config key): |-------|------|----------------|-------------| | `grounding_mode` | string or null | `"strict"`, `"hybrid"`, `null` | RAG grounding policy. `null` removes the key and falls back to `VEKTRA_PROMPT_GROUNDING_MODE`. | | `show_sources` | bool or null | `true`, `false`, `null` | Widget citation visibility (FEAT-014). `null` removes the key and falls back to `VEKTRA_LEARN_SHOW_SOURCES`. The API always returns the full sources list; the flag only instructs the widget whether to render them. Resolution chain: client `data-show-sources` attr > `namespaces.config.show_sources` > `VEKTRA_LEARN_SHOW_SOURCES` env > hardcoded `true`. | +| `citations_enabled` | bool or null | `true`, `false`, `null` | Inline source citations (FEAT-021, advanced pipeline only). When `true`, the LLM is instructed to add `[n]` markers matching the context sources, and each returned source carries a `title` ("filename, p.N") for tooltip rendering. No env var: `null` (or absent) means disabled. Default-off leaves prompts unchanged. | Behavior: - **Partial update**: keys not present in the body are preserved. @@ -340,6 +341,7 @@ Response: | `answer` | LLM-generated answer grounded in sources | | `sources` | Ranked list of source chunks | | `sources[].document_name` | Filename of the source document (e.g. `lecture-07.pdf`), or `null` when the document join returns no row. Soft-deleted documents (REQ-057) keep their citation with an `(archived)` suffix so traceability is preserved. | +| `sources[].title` | FEAT-021: human-readable citation label ("filename, p.N") matching the `[n]` markers in the answer. Set only when the namespace has `citations_enabled`; `null` otherwise. | | `conversation_id` | Echoed back if provided in request | | `context_only` | `true` if LLM failed and raw sources returned | | `no_relevant_context` | `true` if no chunks exceeded relevance threshold | @@ -522,6 +524,7 @@ Response (HTTP 200, JSON): |-------|-------------| | `show_sources` | Server-resolved citation-visibility hint for the widget (FEAT-014). The full `sources` list is always returned regardless; the widget uses the flag to decide whether to render the citations block. See the resolution chain in [Namespaces PATCH](#patch-apiv1adminnamespacesnamespace_idconfig). | | `sources[].document_name` | Filename of the source document. Soft-deleted documents (REQ-057) keep an `(archived)` suffix so traceability is preserved. The field is `null` when the document join returns no row. | +| `sources[].title` | FEAT-021: citation label ("filename, p.N") for the `[n]` markers, `null` unless the namespace has `citations_enabled`. | #### Streaming diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c0753eed..72a32279 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -59,8 +59,10 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| -| `VEKTRA_EMBEDDING_PROVIDER` | str | `sentence-transformers` | Embedding provider implementation | -| `VEKTRA_EMBEDDING_MODEL` | str | `paraphrase-multilingual-MiniLM-L12-v2` | Model name within the selected provider | +| `VEKTRA_EMBEDDING_PROVIDER` | str | `sentence-transformers` | Embedding provider implementation: `sentence-transformers` (in-process), `tei` (remote) | +| `VEKTRA_EMBEDDING_MODEL` | str | `paraphrase-multilingual-MiniLM-L12-v2` | Model name within the selected provider (`sentence-transformers` only; a TEI instance serves one fixed model) | +| `VEKTRA_TEI_URL` | str | `http://localhost:8080` | TEI server base URL (native API, no `/v1` suffix). Used when provider is `tei`. The Qdrant collection is sized from the served model's dimensions at startup | +| `VEKTRA_TEI_API_KEY` | str | - | Bearer token for the TEI embedding server (`--api-key`). Optional | | `VEKTRA_SPARSE_EMBEDDING_PROVIDER` | str | - | Sparse embedding provider: `fastembed-bm25`, `splade` | | `VEKTRA_SPARSE_EMBEDDING_MODEL` | str | - | Sparse embedding model name | @@ -81,6 +83,9 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | `VEKTRA_QUERY_PIPELINE` | str | `advanced` | Pipeline implementation: `simple`, `advanced` | | `VEKTRA_MIN_RELEVANCE_SCORE` | float | `0.15` | Minimum relevance score for chunk inclusion (0.0-1.0). Safety net filter; top-k is the primary control. | | `VEKTRA_CHUNK_DEDUP_ENABLED` | bool | `true` | Deduplicate overlapping adjacent chunks from the same document | +| `VEKTRA_RETRIEVAL_RESCUE_TOP_K` | int | `0` | When the `VEKTRA_MIN_RELEVANCE_SCORE` filter empties the candidate set, keep this many top-scored chunks above the rescue floor instead of refusing. Multi-part and comparative questions get uniformly low reranker scores (each chunk answers only one part), so with the rescue the LLM arbitrates via grounding instead of the query dying at the filter. `0` disables the rescue. Recommended starting point when enabling: `3`. | +| `VEKTRA_RETRIEVAL_RESCUE_FLOOR` | float | `0.02` | Absolute minimum score for rescued chunks (0.0-1.0): candidates below this are never rescued. Only used when `VEKTRA_RETRIEVAL_RESCUE_TOP_K` > 0. Lower values rescue more multi-part questions but feed more irrelevant context to adversarial ones. | +| `VEKTRA_PARENT_EXPANSION_ENABLED` | bool | `false` | Replace retrieved child chunks with their parent chunk text before prompt construction (advanced pipeline only). Requires documents ingested with `VEKTRA_CHUNKING_STRATEGY=dual`. | | `VEKTRA_RESPONSE_TOKEN_RESERVE` | int | `2048` | Tokens reserved for LLM response generation | | `VEKTRA_CONTEXT_CHUNK_RATIO` | float | `0.6` | Fraction of context window allocated to retrieved chunks (0.0-1.0) | | `VEKTRA_PROMPT_TEMPLATES_DIR` | str | - | Directory for custom Jinja2 prompt templates (`system.j2`, `context.j2`, `conversation.j2`). Uses built-in defaults if unset. | @@ -98,9 +103,12 @@ The model name must match the vLLM `--model` path exactly (e.g., `/models/qwen35 | Variable | Type | Default | Description | |----------|------|---------|-------------| | `VEKTRA_RERANK_ENABLED` | bool | `true` | Enable cross-encoder reranking after retrieval | -| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere` | +| `VEKTRA_RERANK_PROVIDER` | str | `cross-encoder` | Reranking provider: `flashrank`, `cross-encoder`, `cohere`, `tei` (remote) | | `VEKTRA_RERANK_MODEL` | str | `BAAI/bge-reranker-v2-m3` | Multilingual reranking model. For English-only lightweight deployments: provider=`flashrank`, model=`ms-marco-MiniLM-L-12-v2` | | `VEKTRA_RERANK_TOP_K` | int | `5` | Final top-k results after reranking | +| `VEKTRA_RERANK_API_KEY` | str | - | API key for API-based providers (`cohere`) | +| `VEKTRA_RERANK_TEI_URL` | str | `http://localhost:8080` | TEI reranker server base URL (one TEI instance per model, e.g. serving `BAAI/bge-reranker-v2-m3`). Used when provider is `tei`. Scores are sigmoid-normalized like the in-process path | +| `VEKTRA_RERANK_TEI_API_KEY` | str | - | Bearer token for the TEI reranker server. Optional | ## Ingestion diff --git a/tests/eval/README.md b/tests/eval/README.md new file mode 100644 index 00000000..e09afe41 --- /dev/null +++ b/tests/eval/README.md @@ -0,0 +1,52 @@ +# RAG evaluation harness (TECH-002) + +Two-stage evaluation against a running Vektra stack. Both scripts are pure HTTP +clients: they require `VEKTRA_API_URL` and `VEKTRA_API_KEY` in the environment. + +```bash +make eval-retrieval # /api/v1/search (no LLM) - hit rate, MRR, precision@k +make eval-e2e # /api/v1/query (full pipeline + LLM) - grounded rate, latency +# extra args: +make eval-retrieval EVAL_ARGS="--dataset tests/eval/dataset-full.jsonl --output tests/eval/results_retrieval_full.jsonl" +``` + +Results are written as JSONL next to the datasets (`results_*.jsonl`, gitignored: +record aggregates in the active `.s2s/plans/` file and in vektra-internal). + +## Datasets + +| File | Namespace | Corpus | +|------|-----------|--------| +| `dataset.jsonl` | `default` | Excerpt corpus, 12 chunks: `costituzione_italiana.md` v2 (6), `udhr_excerpts.md` v2 (4), `sample.pdf` (2). Hit rate saturates here; useful for smoke/regression, not for tuning. | +| `dataset-full.jsonl` | `eval-full` | Full-document corpus, 78 chunks: clean full Italian Constitution (72) + official UDHR English PDF (6). Same 55 questions, discriminative metrics. | + +Both files share the same 55 questions (21 factual, 15 reasoning, 10 multi-chunk, +9 adversarial without ground truth; 37 IT / 18 EN). Entry shape: +`{id, question, expected_keywords, namespace, category, language}`. Relevance is +keyword-based (`expected_keywords`, diacritic-insensitive substring match), so it +is chunking-independent and survives reingestion. + +## Corpus provenance (eval-full) + +- `costituzione-full-clean.md`: full text of the Italian Constitution converted + from Wikisource (`https://it.wikisource.org/api/rest_v1/page/html/Costituzione_della_Repubblica_italiana`, + CC BY-SA), metadata header stripped. Kept outside the repo at + `/mnt/ai/datasets/vektra-eval/` on the dev machine. +- `udhr-en-ohchr.pdf`: official OHCHR English UDHR + (`https://www.ohchr.org/sites/default/files/UDHR/Documents/UDHR_Translations/eng.pdf`). +- Do NOT use the senato.it combined PDF (`costituzione.pdf`, 506 pages): its print + layout breaks pdfplumber extraction (fused words like + `COSTITUZIONEDELLAREPUBBLICAITALIANA`, preserved hyphenation) and invalidates + keyword ground truth. Measured impact: IT hit rate 72% garbled vs 88% clean on + the same questions. Worth keeping in mind as a future "dirty extraction" test + case, but never as a retrieval-quality corpus. + +To rebuild the corpus: download the two sources, then +`scripts/ingest.sh eval-full` for each. + +## Adding questions + +Append JSONL entries with a unique `id`, the target `namespace`, and 1-3 +`expected_keywords` that only appear in the passages that truly answer the +question. Adversarial entries (expected refusal) omit `expected_keywords` and are +reported separately (`has_ground_truth: false`). diff --git a/tests/eval/dataset-full.jsonl b/tests/eval/dataset-full.jsonl new file mode 100644 index 00000000..c486de7f --- /dev/null +++ b/tests/eval/dataset-full.jsonl @@ -0,0 +1,55 @@ +{"id": "IT-F-01", "question": "Quali sono i diritti inviolabili dell'uomo riconosciuti dalla Costituzione italiana?", "expected_keywords": ["diritti inviolabili", "formazioni sociali"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-02", "question": "Su cosa si fonda la Repubblica italiana?", "expected_keywords": ["fondata sul lavoro"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-03", "question": "Cosa stabilisce l'articolo 3 della Costituzione sull'uguaglianza?", "expected_keywords": ["pari dignita' sociale", "senza distinzione"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-04", "question": "A chi appartiene la sovranita' nella Repubblica italiana?", "expected_keywords": ["sovranita' appartiene al popolo"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-05", "question": "Cosa prevede la Costituzione italiana riguardo al diritto al lavoro?", "expected_keywords": ["diritto al lavoro", "condizioni che rendano effettivo"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-06", "question": "Come e' composta la bandiera della Repubblica italiana?", "expected_keywords": ["tricolore", "verde, bianco e rosso"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-07", "question": "Cosa tutela l'articolo 9 della Costituzione?", "expected_keywords": ["cultura", "ricerca scientifica", "paesaggio"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-08", "question": "Cosa stabilisce la Costituzione sulla liberta' personale?", "expected_keywords": ["liberta' personale", "inviolabile", "autorita' giudiziaria"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-09", "question": "Cosa prevede la Costituzione sulle minoranze linguistiche?", "expected_keywords": ["minoranze linguistiche", "tutela"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-10", "question": "Quali sono le garanzie per la liberta' di corrispondenza?", "expected_keywords": ["corrispondenza", "inviolabili", "autorita' giudiziaria"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-11", "question": "Cosa dice la Costituzione sul diritto di riunione?", "expected_keywords": ["riunirsi pacificamente", "senz'armi"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-12", "question": "Cosa prevede l'articolo 32 sulla salute?", "expected_keywords": ["salute", "fondamentale diritto", "cure gratuite"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-13", "question": "Cosa dice la Costituzione sull'obbligo scolastico?", "expected_keywords": ["istruzione inferiore", "obbligatoria e gratuita", "otto anni"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-14", "question": "Qual e' il rapporto tra Stato e Chiesa nella Costituzione?", "expected_keywords": ["indipendenti e sovrani", "Patti Lateranensi"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-F-15", "question": "Cosa prevede l'articolo 21 sulla liberta' di pensiero?", "expected_keywords": ["manifestare liberamente", "pensiero", "stampa"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "IT-R-01", "question": "Qual e' il legame tra il principio di uguaglianza e il diritto al lavoro nella Costituzione?", "expected_keywords": ["uguaglianza", "lavoro"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-02", "question": "Come si conciliano autonomia locale e unita' nazionale nella Costituzione italiana?", "expected_keywords": ["autonomie locali", "una e indivisibile", "decentramento"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-03", "question": "In che modo la Costituzione bilancia liberta' di stampa e intervento giudiziario?", "expected_keywords": ["stampa", "sequestro", "autorita' giudiziaria"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-04", "question": "Quali limiti pone la Costituzione alla liberta' di circolazione?", "expected_keywords": ["circolare", "soggiornare", "sanita'", "sicurezza"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-05", "question": "Come protegge la Costituzione i cittadini capaci ma privi di mezzi economici nel percorso scolastico?", "expected_keywords": ["capaci e meritevoli", "borse di studio"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-06", "question": "In che modo la Costituzione italiana affronta il rapporto tra diritti individuali e solidarieta' sociale?", "expected_keywords": ["diritti inviolabili", "doveri inderogabili", "solidarieta'"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-07", "question": "Quali sono le condizioni per limitare la liberta' personale secondo la Costituzione?", "expected_keywords": ["atto motivato", "autorita' giudiziaria", "casi e modi previsti dalla legge"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-08", "question": "Come affronta la Costituzione la liberta' religiosa per le confessioni diverse dalla cattolica?", "expected_keywords": ["confessioni religiose", "egualmente libere", "intese"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-09", "question": "Qual e' il rapporto tra trattamento sanitario obbligatorio e rispetto della persona nella Costituzione?", "expected_keywords": ["trattamento sanitario", "disposizione di legge", "rispetto della persona umana"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "IT-R-10", "question": "Come la Costituzione tutela l'ambiente rispetto all'attivita' economica?", "expected_keywords": ["ambiente", "biodiversita'", "attivita' economica", "fini sociali e ambientali"], "namespace": "eval-full", "category": "reasoning", "language": "it"} +{"id": "EN-F-01", "question": "What does Article 1 of the UDHR say about human beings?", "expected_keywords": ["born free and equal", "dignity and rights"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-02", "question": "What does the UDHR say about slavery?", "expected_keywords": ["slavery", "servitude", "prohibited"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-03", "question": "What rights does Article 19 of the UDHR protect?", "expected_keywords": ["freedom of opinion", "expression", "information"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-04", "question": "What does the UDHR say about education?", "expected_keywords": ["right to education", "free", "compulsory"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-F-05", "question": "What does Article 9 of the UDHR state about arrest?", "expected_keywords": ["arbitrary arrest", "detention", "exile"], "namespace": "eval-full", "category": "factual", "language": "en"} +{"id": "EN-R-01", "question": "How does the UDHR connect the right to work with human dignity?", "expected_keywords": ["right to work", "remuneration", "dignity"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-02", "question": "What is the relationship between marriage and equality in the UDHR?", "expected_keywords": ["marry", "equal rights", "free and full consent"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-03", "question": "How does the UDHR protect against discrimination?", "expected_keywords": ["without distinction", "equal protection", "discrimination"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-04", "question": "What protections does the UDHR provide for motherhood and childhood?", "expected_keywords": ["motherhood", "childhood", "special care"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "EN-R-05", "question": "How does the UDHR balance individual rights with government authority?", "expected_keywords": ["will of the people", "elections", "universal", "suffrage"], "namespace": "eval-full", "category": "reasoning", "language": "en"} +{"id": "MC-01", "question": "Come si confrontano le garanzie sulla liberta' personale nella Costituzione italiana e nella Dichiarazione Universale?", "expected_keywords": ["liberta' personale", "arbitrary arrest"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-02", "question": "Quali analogie esistono tra l'articolo 3 della Costituzione italiana e l'articolo 2 della UDHR sulla non discriminazione?", "expected_keywords": ["senza distinzione", "without distinction"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-03", "question": "How do both the Italian Constitution and the UDHR address freedom of religion?", "expected_keywords": ["fede religiosa", "freedom of thought"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-04", "question": "Quali somiglianze ci sono nel trattamento del diritto all'istruzione tra Costituzione italiana e UDHR?", "expected_keywords": ["istruzione", "education", "obbligatoria"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-05", "question": "How do both documents address the right to work and fair employment?", "expected_keywords": ["diritto al lavoro", "right to work"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-06", "question": "Quali differenze ci sono nella tutela della famiglia tra la Costituzione italiana e la UDHR?", "expected_keywords": ["famiglia", "matrimonio", "family"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-07", "question": "Come affrontano entrambi i documenti il diritto alla privacy e all'inviolabilita' del domicilio?", "expected_keywords": ["domicilio", "privacy", "correspondence"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-08", "question": "What similarities exist in how both documents treat freedom of expression and press?", "expected_keywords": ["pensiero", "expression", "stampa"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "MC-09", "question": "Come entrambi i documenti proteggono il diritto alla salute e a un tenore di vita adeguato?", "expected_keywords": ["salute", "health", "well-being"], "namespace": "eval-full", "category": "multi-chunk", "language": "it"} +{"id": "MC-10", "question": "How do the Italian Constitution and UDHR compare on the presumption of innocence and fair trial?", "expected_keywords": ["presumed innocent", "autorita' giudiziaria"], "namespace": "eval-full", "category": "multi-chunk", "language": "en"} +{"id": "ADV-01", "question": "Cosa dice la Costituzione italiana sul diritto di voto degli stranieri?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-02", "question": "Qual e' la pena prevista dalla Costituzione per chi viola i diritti fondamentali?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-03", "question": "What does the UDHR say about the right to bear arms?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-04", "question": "Cosa stabilisce la Costituzione italiana sulla pena di morte?", "expected_keywords": ["pena di morte", "non e' ammessa"], "namespace": "eval-full", "category": "factual", "language": "it"} +{"id": "ADV-05", "question": "What is the UDHR's position on artificial intelligence and data privacy?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-06", "question": "Quanti articoli ha la Costituzione italiana?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-07", "question": "Who wrote the Universal Declaration of Human Rights?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-08", "question": "Cosa dice la Costituzione sul numero massimo di mandati presidenziali?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} +{"id": "ADV-09", "question": "What penalties does the UDHR prescribe for nations that violate human rights?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "en"} +{"id": "ADV-10", "question": "Come regolamenta la Costituzione italiana l'uso dei social media?", "expected_keywords": [], "namespace": "eval-full", "category": "adversarial", "language": "it"} diff --git a/uv.lock b/uv.lock index 985f6c20..c7dff6be 100644 --- a/uv.lock +++ b/uv.lock @@ -4616,7 +4616,8 @@ dependencies = [ { name = "safetensors", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, { name = "torch", version = "2.10.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, - { name = "torchvision", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "torchvision", version = "0.25.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "torchvision", version = "0.25.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d7/2c/593109822fe735e637382aca6640c1102c19797f7791f1fd1dab2d6c3cb1/timm-1.0.25.tar.gz", hash = "sha256:47f59fc2754725735cc81bb83bcbfce5bec4ebd5d4bb9e69da57daa92fcfa768", size = 2414743, upload-time = "2026-02-23T16:49:00.137Z" } wheels = [ @@ -4766,34 +4767,76 @@ wheels = [ [[package]] name = "torchvision" version = "0.25.0" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'darwin'", +] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pillow", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "numpy", marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "pillow", marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, { name = "torch", version = "2.10.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, - { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'win32') or (platform_machine == 's390x' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'win32')" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/56/3a/6ea0d73f49a9bef38a1b3a92e8dd455cea58470985d25635beab93841748/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2abe430c90b1d5e552680037d68da4eb80a5852ebb1c811b2b89d299b10573b", size = 1874920, upload-time = "2026-01-21T16:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/51/f8/c0e1ef27c66e15406fece94930e7d6feee4cb6374bbc02d945a630d6426e/torchvision-0.25.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b75deafa2dfea3e2c2a525559b04783515e3463f6e830cb71de0fb7ea36fe233", size = 2344556, upload-time = "2026-01-21T16:27:40.125Z" }, - { url = "https://files.pythonhosted.org/packages/68/2f/f24b039169db474e8688f649377de082a965fbf85daf4e46c44412f1d15a/torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f25aa9e380865b11ea6e9d99d84df86b9cc959f1a007cd966fc6f1ab2ed0e248", size = 8072351, upload-time = "2026-01-21T16:27:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/16/8f650c2e288977cf0f8f85184b90ee56ed170a4919347fc74ee99286ed6f/torchvision-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9c55ae8d673ab493325d1267cbd285bb94d56f99626c00ac4644de32a59ede3", size = 4303059, upload-time = "2026-01-21T16:27:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5b/1562a04a6a5a4cf8cf40016a0cdeda91ede75d6962cff7f809a85ae966a5/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:24e11199e4d84ba9c5ee7825ebdf1cd37ce8deec225117f10243cae984ced3ec", size = 1874918, upload-time = "2026-01-21T16:27:39.02Z" }, - { url = "https://files.pythonhosted.org/packages/36/b1/3d6c42f62c272ce34fcce609bb8939bdf873dab5f1b798fd4e880255f129/torchvision-0.25.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f271136d2d2c0b7a24c5671795c6e4fd8da4e0ea98aeb1041f62bc04c4370ef", size = 2309106, upload-time = "2026-01-21T16:27:30.624Z" }, - { url = "https://files.pythonhosted.org/packages/c7/60/59bb9c8b67cce356daeed4cb96a717caa4f69c9822f72e223a0eae7a9bd9/torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:855c0dc6d37f462482da7531c6788518baedca1e0847f3df42a911713acdfe52", size = 8071522, upload-time = "2026-01-21T16:27:29.392Z" }, - { url = "https://files.pythonhosted.org/packages/32/a5/9a9b1de0720f884ea50dbf9acb22cbe5312e51d7b8c4ac6ba9b51efd9bba/torchvision-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:cef0196be31be421f6f462d1e9da1101be7332d91984caa6f8022e6c78a5877f", size = 4321911, upload-time = "2026-01-21T16:27:35.195Z" }, - { url = "https://files.pythonhosted.org/packages/52/99/dca81ed21ebaeff2b67cc9f815a20fdaa418b69f5f9ea4c6ed71721470db/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a8f8061284395ce31bcd460f2169013382ccf411148ceb2ee38e718e9860f5a7", size = 1896209, upload-time = "2026-01-21T16:27:32.159Z" }, - { url = "https://files.pythonhosted.org/packages/28/cc/2103149761fdb4eaed58a53e8437b2d716d48f05174fab1d9fcf1e2a2244/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:146d02c9876858420adf41f3189fe90e3d6a409cbfa65454c09f25fb33bf7266", size = 2310735, upload-time = "2026-01-21T16:27:22.327Z" }, - { url = "https://files.pythonhosted.org/packages/76/ad/f4c985ad52ddd3b22711c588501be1b330adaeaf6850317f66751711b78c/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c4d395cb2c4a2712f6eb93a34476cdf7aae74bb6ea2ea1917f858e96344b00aa", size = 8089557, upload-time = "2026-01-21T16:27:27.666Z" }, - { url = "https://files.pythonhosted.org/packages/63/cc/0ea68b5802e5e3c31f44b307e74947bad5a38cc655231d845534ed50ddb8/torchvision-0.25.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5e6b449e9fa7d642142c0e27c41e5a43b508d57ed8e79b7c0a0c28652da8678c", size = 4344260, upload-time = "2026-01-21T16:27:17.018Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1f/fa839532660e2602b7e704d65010787c5bb296258b44fa8b9c1cd6175e7d/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:620a236288d594dcec7634c754484542dc0a5c1b0e0b83a34bda5e91e9b7c3a1", size = 1896193, upload-time = "2026-01-21T16:27:24.785Z" }, - { url = "https://files.pythonhosted.org/packages/80/ed/d51889da7ceaf5ff7a0574fb28f9b6b223df19667265395891f81b364ab3/torchvision-0.25.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b5e7f50002a8145a98c5694a018e738c50e2972608310c7e88e1bd4c058f6ce", size = 2309331, upload-time = "2026-01-21T16:27:19.97Z" }, - { url = "https://files.pythonhosted.org/packages/90/a5/f93fcffaddd8f12f9e812256830ec9c9ca65abbf1bc369379f9c364d1ff4/torchvision-0.25.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:632db02300e83793812eee4f61ae6a2686dab10b4cfd628b620dc47747aa9d03", size = 8088713, upload-time = "2026-01-21T16:27:15.281Z" }, - { url = "https://files.pythonhosted.org/packages/1f/eb/d0096eed5690d962853213f2ee00d91478dfcb586b62dbbb449fb8abc3a6/torchvision-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1abd5ed030c708f5dbf4812ad5f6fbe9384b63c40d6bd79f8df41a4a759a917", size = 4325058, upload-time = "2026-01-21T16:27:26.165Z" }, - { url = "https://files.pythonhosted.org/packages/97/36/96374a4c7ab50dea9787ce987815614ccfe988a42e10ac1a2e3e5b60319a/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad9a8a5877782944d99186e4502a614770fe906626d76e9cd32446a0ac3075f2", size = 1896207, upload-time = "2026-01-21T16:27:23.383Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e2/7abb10a867db79b226b41da419b63b69c0bd5b82438c4a4ed50e084c552f/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:40a122c3cf4d14b651f095e0f672b688dde78632783fc5cd3d4d5e4f6a828563", size = 2310741, upload-time = "2026-01-21T16:27:18.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/e6/0927784e6ffc340b6676befde1c60260bd51641c9c574b9298d791a9cda4/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:846890161b825b38aa85fc37fb3ba5eea74e7091ff28bab378287111483b6443", size = 8089772, upload-time = "2026-01-21T16:27:14.048Z" }, - { url = "https://files.pythonhosted.org/packages/b6/37/e7ca4ec820d434c0f23f824eb29f0676a0c3e7a118f1514f5b949c3356da/torchvision-0.25.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f07f01d27375ad89d72aa2b3f2180f07da95dd9d2e4c758e015c0acb2da72977", size = 4425879, upload-time = "2026-01-21T16:27:12.579Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:724f212a58a0d0d758649ce288601056b5f46a01de545702f42bccc5b25cb0cc", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6c444119c6af8fa48b79c59f1529fc15a45a2bbf823cf85f851e7203d84f727f", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:80895a40faa5783ce19ed5a692a54062c7888a385490e866324355f8d59b2eb3", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a64ae543fe0a057a93954af94c239629723db196ed65090e6775f136726b22f8", upload-time = "2026-01-20T18:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b7df71942e31b74a5161dea3452da35e5fb1d36c752aedf61f2dd3c7be35739a", upload-time = "2026-01-20T18:32:31Z" }, +] + +[[package]] +name = "torchvision" +version = "0.25.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version >= '3.15' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.14.*' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "numpy", marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, + { name = "pillow", marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, + { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:727334e9a721cfc1ac296ce0bf9e69d9486821bfa5b1e75a8feb6f78041db481", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c1be164e93c68b2dbf460fd58975377c892dbcf3358fb72941709c3857351bba", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2d444009c0956669ada149f61ed78f257c1cc96d259efa6acf3929ca96ceb3f0", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fe54cbd5942cd0b26a90f1748f0d4421caf67be35c281c6c3b8573733a03d630", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:90eec299e1f82cfaf080ccb789df3838cb9a54b57e2ebe33852cd392c692de5c", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:783c8fc580bbfc159bff52f4f72cdd538e42b32956e70dffa42b940db114e151", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e985e12a9a232618e5a43476de5689e4b14989f5da6b93909c57afa57ec27012", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:813f0106eb3e268f3783da67b882458e544c6fb72f946e6ca64b5ed4e62c6a77", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:9212210f417888e6261c040495180f053084812cf873dedba9fc51ff4b24b2d3", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0c2d0da9bc011a0fde1d125af396a8fbe94d99becf9d313764f24ca7657a3448", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4d72a57a8f0b5146e26dac1fbfa2c905280cd04f5fcb23b9c56253506b683aeb", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:499eae1e535766391b6ee2d1e6e841239c20e2e6d88203a15b8f9f8d60a1f8bd", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7d47d544899fabac52ebe0d4812975608fd7ab79a3d7fb6383275eb667e33f53", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9511339b3b5eb75229e0b5041202e8aed9bef3b1de3a715b9fb319c9e97688fd", upload-time = "2026-01-20T18:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.25.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:fb9f07f6a10f0ac24ac482ae68c6df99110b74a0d80a4c64fddc9753267d8815", upload-time = "2026-01-20T18:32:30Z" }, ] [[package]] @@ -5056,7 +5099,7 @@ wheels = [ [[package]] name = "vektra-admin" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-admin" } dependencies = [ { name = "argon2-cffi" }, @@ -5105,7 +5148,7 @@ dev = [ [[package]] name = "vektra-analytics" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-analytics" } dependencies = [ { name = "fastapi" }, @@ -5140,7 +5183,7 @@ dev = [ [[package]] name = "vektra-app" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-app" } dependencies = [ { name = "alembic" }, @@ -5189,10 +5232,11 @@ dev = [ [[package]] name = "vektra-core" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-core" } dependencies = [ { name = "fastapi" }, + { name = "httpx" }, { name = "jinja2" }, { name = "litellm" }, { name = "presidio-analyzer" }, @@ -5213,6 +5257,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "fastapi", specifier = ">=0.115" }, + { name = "httpx", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "litellm", specifier = ">=1.83.10" }, { name = "presidio-analyzer", specifier = ">=2.2" }, @@ -5232,11 +5277,12 @@ dev = [ [[package]] name = "vektra-index" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-index" } dependencies = [ { name = "asyncpg" }, { name = "fastapi" }, + { name = "httpx" }, { name = "pgvector" }, { name = "pydantic" }, { name = "sentence-transformers" }, @@ -5267,6 +5313,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.29" }, { name = "fastapi", specifier = ">=0.115" }, { name = "fastembed", marker = "extra == 'sparse'", specifier = ">=0.4" }, + { name = "httpx", specifier = ">=0.27" }, { name = "pgvector", specifier = ">=0.3" }, { name = "pydantic", specifier = ">=2,<3" }, { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = ">=1.12" }, @@ -5287,7 +5334,7 @@ dev = [ [[package]] name = "vektra-ingest" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-ingest" } dependencies = [ { name = "arq" }, @@ -5307,6 +5354,8 @@ dependencies = [ [package.optional-dependencies] ocr = [ + { name = "torchvision", version = "0.25.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and platform_machine != 's390x' and sys_platform == 'darwin'" }, + { name = "torchvision", version = "0.25.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and platform_machine != 's390x') or platform_machine == 's390x' or sys_platform != 'darwin'" }, { name = "unstructured", extra = ["pdf"] }, ] @@ -5333,6 +5382,7 @@ requires-dist = [ { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0" }, { name = "structlog", specifier = ">=24.0" }, { name = "tiktoken", specifier = ">=0.7" }, + { name = "torchvision", marker = "extra == 'ocr'", specifier = ">=0.25,<0.26", index = "https://download.pytorch.org/whl/cpu" }, { name = "unstructured", extras = ["pdf"], marker = "extra == 'ocr'", specifier = ">=0.15" }, { name = "vektra-shared", editable = "vektra-shared" }, ] @@ -5349,7 +5399,7 @@ dev = [ [[package]] name = "vektra-learn" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-learn" } dependencies = [ { name = "fastapi" }, @@ -5386,7 +5436,7 @@ dev = [ [[package]] name = "vektra-shared" -version = "0.5.1" +version = "0.6.0" source = { editable = "vektra-shared" } dependencies = [ { name = "asyncpg" }, diff --git a/vektra-admin/pyproject.toml b/vektra-admin/pyproject.toml index 84be6c56..70c64737 100644 --- a/vektra-admin/pyproject.toml +++ b/vektra-admin/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-admin" -version = "0.5.1" +version = "0.6.0" description = "System administration interface: health, API key management, audit log" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-admin/src/vektra_admin/api.py b/vektra-admin/src/vektra_admin/api.py index aa7ae1b5..a98caa24 100644 --- a/vektra-admin/src/vektra_admin/api.py +++ b/vektra-admin/src/vektra_admin/api.py @@ -63,6 +63,7 @@ } ALLOWED_CONFIG_TYPES: dict[str, type] = { "show_sources": bool, + "citations_enabled": bool, } assert ALLOWED_CONFIG_VALUES.keys().isdisjoint(ALLOWED_CONFIG_TYPES.keys()), ( "Each config key must appear in exactly one of " @@ -660,6 +661,8 @@ async def get_namespace_config( "show_sources": getattr( request.app.state, "learn_show_sources_default", True ), + # FEAT-021: no env var, hardcoded default off + "citations_enabled": False, }, ) return NamespaceConfigDetailResponse( diff --git a/vektra-admin/tests/test_integration.py b/vektra-admin/tests/test_integration.py index 50c550bf..86df7aef 100644 --- a/vektra-admin/tests/test_integration.py +++ b/vektra-admin/tests/test_integration.py @@ -912,9 +912,14 @@ async def test_namespace_config_get_returns_empty_with_resolved_defaults( # Hardcoded resolver defaults: grounding_mode="strict", show_sources=True. # In CI the env vars may shift these; assert the keys are present and bool/string-typed # rather than a fixed value, so the test stays independent of test env config. - assert set(body["resolved"]) == {"grounding_mode", "show_sources"} + assert set(body["resolved"]) == { + "grounding_mode", + "show_sources", + "citations_enabled", + } assert body["resolved"]["grounding_mode"] in {"strict", "hybrid"} assert isinstance(body["resolved"]["show_sources"], bool) + assert body["resolved"]["citations_enabled"] is False # FEAT-021 default async def test_namespace_config_get_reflects_stored_overrides( @@ -981,3 +986,47 @@ async def test_namespace_config_get_requires_admin_scope( headers={"Authorization": f"Bearer {query_key}"}, ) assert resp.status_code == 403, resp.text + + +# --------------------------------------------------------------------------- +# FEAT-021: citations_enabled per-namespace setting +# --------------------------------------------------------------------------- + + +async def test_namespace_config_patch_sets_citations_enabled( + client, bootstrap_key, fresh_engine +): + """PATCH with citations_enabled=true persists as a bool in namespaces.config.""" + admin_key = await _create_admin_key(client, bootstrap_key) + ns_id = "feat21-citations-true" + await _seed_namespace(fresh_engine, ns_id) + + resp = await client.patch( + f"/api/v1/admin/namespaces/{ns_id}/config", + json={"citations_enabled": True}, + headers={"Authorization": f"Bearer {admin_key}"}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["config"] == {"citations_enabled": True} + + await asyncio.sleep(0.2) + stored = await _read_namespace_config(fresh_engine, ns_id) + assert stored == {"citations_enabled": True} + + +async def test_namespace_config_patch_rejects_citations_enabled_non_bool( + client, bootstrap_key, fresh_engine +): + """citations_enabled is strictly bool: strings are rejected with ERR-ADMIN-007.""" + admin_key = await _create_admin_key(client, bootstrap_key) + ns_id = "feat21-citations-nonbool" + await _seed_namespace(fresh_engine, ns_id) + + resp = await client.patch( + f"/api/v1/admin/namespaces/{ns_id}/config", + json={"citations_enabled": "yes"}, + headers={"Authorization": f"Bearer {admin_key}"}, + ) + assert resp.status_code == 400, resp.text + err = resp.json()["detail"]["error"] + assert err["code"] == "ERR-ADMIN-007" diff --git a/vektra-analytics/pyproject.toml b/vektra-analytics/pyproject.toml index d08dd8e2..7e97aee9 100644 --- a/vektra-analytics/pyproject.toml +++ b/vektra-analytics/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-analytics" -version = "0.5.1" +version = "0.6.0" description = "QueryTrace storage, metrics aggregation, and reporting API" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-app/pyproject.toml b/vektra-app/pyproject.toml index f3e42aac..a4b2bde3 100644 --- a/vektra-app/pyproject.toml +++ b/vektra-app/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-app" -version = "0.5.1" +version = "0.6.0" description = "FastAPI application assembly and startup validation (ARCH-015, ARCH-057)" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-app/src/vektra_app/__init__.py b/vektra-app/src/vektra_app/__init__.py index 5f0ed55f..ea690d8e 100644 --- a/vektra-app/src/vektra_app/__init__.py +++ b/vektra-app/src/vektra_app/__init__.py @@ -1,3 +1,3 @@ # vektra-app: FastAPI assembly and startup validation (ARCH-015) -__version__ = "0.4.0-dev" +__version__ = "0.6.0" diff --git a/vektra-app/src/vektra_app/main.py b/vektra-app/src/vektra_app/main.py index 25a825fa..30e8f1bb 100644 --- a/vektra-app/src/vektra_app/main.py +++ b/vektra-app/src/vektra_app/main.py @@ -31,6 +31,7 @@ from vektra_shared.config import QueryPipelineConfig, VektraSettings from vektra_shared.db import init_db from vektra_shared.errors import ERR_CONFIG_001, ErrorCategory, ErrorResponse +from vektra_shared.protocols import EmbeddingProvider from vektra_shared.registry import ProviderRegistry from vektra_shared.startup import ( StartupValidationError, @@ -126,15 +127,31 @@ async def _step_5_register_providers( registry.register("llm", "default", llm_provider) # --- Embedding --- - from vektra_index.providers.sentence_transformers import ( - SentenceTransformersProvider, - ) + embedding_provider: EmbeddingProvider + if settings.embedding_provider == "tei": + from vektra_index.providers.tei import TEIEmbeddingProvider - embedding_provider = SentenceTransformersProvider( - model_name=settings.embedding_model - ) - registry.register("embedding", "default", embedding_provider) - registry.register("embedding", "sentence-transformers", embedding_provider) + embedding_provider = TEIEmbeddingProvider( + url=settings.tei_url, + api_key=settings.tei_api_key, + ) + registry.register("embedding", "default", embedding_provider) + registry.register("embedding", "tei", embedding_provider) + log.info( + "embedding_registered", + provider="tei", + url=_redact_url(settings.tei_url), + ) + else: + from vektra_index.providers.sentence_transformers import ( + SentenceTransformersProvider, + ) + + embedding_provider = SentenceTransformersProvider( + model_name=settings.embedding_model + ) + registry.register("embedding", "default", embedding_provider) + registry.register("embedding", "sentence-transformers", embedding_provider) # --- Vector store --- from vektra_index.adapters import VectorStoreServiceAdapter @@ -168,6 +185,10 @@ async def _step_5_register_providers( api_key=settings.qdrant_api_key, collection_name=settings.qdrant_collection, active_index_version=settings.active_index_version, + # FEAT-024: size the collection from the active embedding model + # instead of the hardcoded 384 default (latent bug for any + # non-384 model; bge-m3 via TEI is 1024). + dense_dimensions=embedding_provider.dimensions(), ) registry.register("vector_store", "default", qdrant_provider) registry.register("vector_store", "qdrant", qdrant_provider) @@ -246,6 +267,9 @@ async def _step_5_register_providers( pipeline_config = QueryPipelineConfig() reranker = create_reranker(pipeline_config.rerank) + if reranker is not None: + # Registered so the lifespan teardown can close remote clients (FEAT-024) + registry.register("reranker", "default", reranker) # --- Query pipeline (Phase 2: select simple or advanced) --- templates_dir = ( @@ -538,6 +562,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Shutdown log.info("shutdown_started") + # Close remote provider HTTP clients (TEI, FEAT-024) best-effort + for category in ("embedding", "reranker"): + try: + provider = registry.get(category, "default") + except Exception: + continue + aclose = getattr(provider, "aclose", None) + if aclose is not None: + try: + await aclose() + except Exception as exc: + log.warning("provider_close_failed", category=category, error=str(exc)) from vektra_shared.db import get_engine engine = get_engine() diff --git a/vektra-core/pyproject.toml b/vektra-core/pyproject.toml index f1dabe5b..5946fbfe 100644 --- a/vektra-core/pyproject.toml +++ b/vektra-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-core" -version = "0.5.1" +version = "0.6.0" description = "RAG engine, LLM abstraction, and conversation management" readme = "README.md" requires-python = ">=3.12" @@ -14,6 +14,7 @@ dependencies = [ "presidio-analyzer>=2.2", "presidio-anonymizer>=2.2", "rerankers[flashrank]>=0.5", + "httpx>=0.27", ] [build-system] diff --git a/vektra-core/src/vektra_core/advanced_pipeline.py b/vektra-core/src/vektra_core/advanced_pipeline.py index 68cee4aa..14dfec3d 100644 --- a/vektra-core/src/vektra_core/advanced_pipeline.py +++ b/vektra-core/src/vektra_core/advanced_pipeline.py @@ -8,6 +8,8 @@ 4. rerank - cross-encoder reranking (skip if not configured) 5. retrieval_filter - score threshold + overlap dedup (ARCH-056) 6. post_retrieval - SafeguardHook.post_retrieval() (DEBT-003) + 6.5 parent_expansion - replace child text with parent chunk text (FEAT-017, + skip unless VEKTRA_PARENT_EXPANSION_ENABLED) 7. build_prompt - token budget + Jinja2 rendering (ARCH-055) 8. llm_call - LLM with graceful degradation (ARCH-043) 9. pre_response - SafeguardHook.pre_response() (ARCH-049) @@ -16,6 +18,7 @@ from __future__ import annotations import asyncio +import dataclasses import hashlib import time from collections.abc import AsyncGenerator, AsyncIterator @@ -36,7 +39,7 @@ _history_to_messages, _trace_to_dict, ) -from vektra_core.reranker import RerankerService +from vektra_core.reranker import RerankerProtocol from vektra_core.templates import TemplateRenderer from vektra_shared.config import LLMConfig, QueryPipelineConfig from vektra_shared.protocols import ( @@ -84,7 +87,7 @@ def __init__( renderer: TemplateRenderer, pipeline_config: QueryPipelineConfig, sparse_embedding: SparseEmbeddingProvider | None = None, - reranker: RerankerService | None = None, + reranker: RerankerProtocol | None = None, ) -> None: self._embedding = embedding self._vector_store = vector_store @@ -99,6 +102,7 @@ def __init__( self._rewrite_enabled = pipeline_config.rewrite.enabled self._eval_mode = pipeline_config.eval_mode self._debug_log_queries = pipeline_config.debug_log_queries + self._parent_expansion = pipeline_config.parent_expansion_enabled # -- Helpers (delegating to shared module-level functions) -- @@ -345,10 +349,12 @@ async def _run_pre_llm_steps( # Step 5: Retrieval filter (ARCH-056) t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -359,6 +365,7 @@ async def _run_pre_llm_steps( "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) @@ -407,6 +414,20 @@ async def _run_pre_llm_steps( if not no_relevant_context and not filtered: no_relevant_context = True + # Step 6.5: Parent chunk expansion (FEAT-017) + if self._parent_expansion and filtered: + t0 = time.monotonic() + filtered, expansion_meta = await self._expand_parents( + query.namespace, filtered + ) + steps.append( + StepTrace( + name="parent_expansion", + duration_ms=_elapsed_ms(t0), + metadata=expansion_meta, + ) + ) + return ( steps, filtered, @@ -416,11 +437,80 @@ async def _run_pre_llm_steps( history, ) + async def _expand_parents( + self, + namespace: str, + results: list[SearchResult], + ) -> tuple[list[SearchResult], dict[str, object]]: + """Replace child chunk text with the parent chunk text (FEAT-017). + + Children of the same parent collapse into a single result (the + highest-scored one, since results arrive score-ordered): the parent + text already contains all its children. Runs before token budgeting + (ARCH-055) so the budget sees the expanded text. Results keep the + child's chunk_id and score for trace comparability. + """ + parent_ids = list(dict.fromkeys(r.parent_id for r in results if r.parent_id)) + if not parent_ids: + return results, {"children_expanded": 0, "parents_fetched": 0} + + try: + parents = await self._vector_store.retrieve( + namespace=namespace, chunk_ids=parent_ids + ) + except Exception as exc: + log.warning("parent_expansion_failed", error=str(exc)) + return results, {"skipped": True, "error": str(exc)} + + parent_text = {p.chunk_id: p.text_snippet for p in parents} + expanded: list[SearchResult] = [] + seen_parents: set[str] = set() + children_expanded = 0 + siblings_merged = 0 + for r in results: + if r.parent_id and r.parent_id in parent_text: + if r.parent_id in seen_parents: + siblings_merged += 1 + continue + seen_parents.add(r.parent_id) + children_expanded += 1 + expanded.append( + dataclasses.replace(r, text_snippet=parent_text[r.parent_id]) + ) + else: + expanded.append(r) + + return expanded, { + "children_expanded": children_expanded, + "siblings_merged": siblings_merged, + "parents_fetched": len(parents), + "after_expansion": len(expanded), + } + + @staticmethod + def _chunk_title( + result: SearchResult, + name_map: dict[str, str], + ) -> str | None: + """Human-readable source title for citations (FEAT-021). + + Prefers "document_name, p.N"; falls back to the raw source_file + from chunk metadata, then None (attribute omitted in the template). + """ + name = name_map.get(str(result.document_id)) or result.metadata.get( + "source_file" + ) + if not name: + return None + page = result.metadata.get("page") + return f"{name}, p.{page}" if page is not None else str(name) + def _build_prompt( self, query: QueryRequest, filtered: list[SearchResult], history: list[dict[str, str | None]], + name_map: dict[str, str] | None = None, ) -> tuple[ list[Message], list[SearchResult], list[dict[str, str | None]], StepTrace ]: @@ -432,6 +522,7 @@ def _build_prompt( namespace=query.namespace, grounding_mode=query.grounding_mode, has_context=has_context, + citations_enabled=query.citations_enabled, ) system_tokens = self._count_tokens(system_text) question_tokens = self._count_tokens(query.question) @@ -457,8 +548,20 @@ def _build_prompt( selected_history = [history[i] for i in selected_history_idx] if selected_chunks: + _names = name_map or {} context_text = self._renderer.render_context( - [{"text": r.text_snippet, "score": r.score} for r in selected_chunks] + [ + { + "text": r.text_snippet, + "score": r.score, + "title": ( + self._chunk_title(r, _names) + if query.citations_enabled + else None + ), + } + for r in selected_chunks + ] ) user_content = f"{context_text}\n\nQuestion: {query.question}" else: @@ -538,15 +641,25 @@ async def execute( no_relevant_context=no_relevant_context and not safeguard_blocked, ), trace - # Step 7: Build prompt + # Step 7: Build prompt (citations need document names up front, FEAT-021) + prompt_name_map: dict[str, str] = {} + if query.citations_enabled and filtered: + prompt_name_map = await _fetch_document_names( + [r.document_id for r in filtered] + ) messages, selected_chunks, _, prompt_step = self._build_prompt( - query, filtered, history + query, filtered, history, name_map=prompt_name_map ) steps.append(prompt_step) # Sources from budget-selected chunks only (not all filtered) t0 = time.monotonic() - name_map = await _fetch_document_names([r.document_id for r in selected_chunks]) + if query.citations_enabled: + name_map = prompt_name_map + else: + name_map = await _fetch_document_names( + [r.document_id for r in selected_chunks] + ) steps.append( StepTrace( name="document_names", @@ -566,6 +679,9 @@ async def execute( citation_id=uuid4(), document_version=r.document_version, document_name=name_map.get(str(r.document_id)), + title=( + self._chunk_title(r, name_map) if query.citations_enabled else None + ), ) for r in selected_chunks ] @@ -705,9 +821,14 @@ async def _stream( yield QueryChunk(type="done", data="") return - # Step 7: Build prompt + # Step 7: Build prompt (citations need document names up front, FEAT-021) + prompt_name_map: dict[str, str] = {} + if query.citations_enabled and filtered: + prompt_name_map = await _fetch_document_names( + [r.document_id for r in filtered] + ) messages, selected_chunks, _, prompt_step = self._build_prompt( - query, filtered, history + query, filtered, history, name_map=prompt_name_map ) steps.append(prompt_step) @@ -806,7 +927,12 @@ async def _stream( # Yield sources (only budget-selected chunks, not all filtered) t0 = time.monotonic() - name_map = await _fetch_document_names([r.document_id for r in selected_chunks]) + if query.citations_enabled: + name_map = prompt_name_map + else: + name_map = await _fetch_document_names( + [r.document_id for r in selected_chunks] + ) steps.append( StepTrace( name="document_names", @@ -826,6 +952,9 @@ async def _stream( "citation_id": str(uuid4()), "document_version": r.document_version, "document_name": name_map.get(str(r.document_id)), + "title": ( + self._chunk_title(r, name_map) if query.citations_enabled else None + ), } for r in selected_chunks ] diff --git a/vektra-core/src/vektra_core/api.py b/vektra-core/src/vektra_core/api.py index 023c563d..cd6fad6a 100644 --- a/vektra-core/src/vektra_core/api.py +++ b/vektra-core/src/vektra_core/api.py @@ -38,7 +38,7 @@ ErrorResponse, http_status_for, ) -from vektra_shared.namespace import resolve_grounding_mode +from vektra_shared.namespace import resolve_citations_enabled, resolve_grounding_mode from vektra_shared.types import ( QueryChunk, QueryRequest, @@ -77,6 +77,7 @@ class SourceRefBody(BaseModel): citation_id: UUID document_version: int = 1 document_name: str | None = None # FEAT-012 + title: str | None = None # FEAT-021: set when the namespace cites sources class QueryResponseBody(BaseModel): @@ -270,6 +271,11 @@ async def query( else: grounding_mode = _default_mode + # Resolve citations: namespace config > default false (FEAT-021) + citations_enabled = False + if db_factory: + citations_enabled = await resolve_citations_enabled(body.namespace, db_factory) + query_req = QueryRequest( question=body.question, namespace=body.namespace, @@ -277,6 +283,7 @@ async def query( top_k=body.top_k, stream=use_stream, grounding_mode=grounding_mode, + citations_enabled=citations_enabled, ) if use_stream: @@ -356,6 +363,7 @@ async def query( citation_id=s.citation_id, document_version=s.document_version, document_name=s.document_name, + title=s.title, # FEAT-021 ) for s in response.sources ], diff --git a/vektra-core/src/vektra_core/pipeline.py b/vektra-core/src/vektra_core/pipeline.py index cd950372..2e3ea76c 100644 --- a/vektra-core/src/vektra_core/pipeline.py +++ b/vektra-core/src/vektra_core/pipeline.py @@ -143,20 +143,40 @@ def _apply_retrieval_filter( results: list[SearchResult], min_score: float, dedup_enabled: bool, -) -> list[SearchResult]: + rescue_top_k: int = 0, + rescue_floor: float = 0.0, +) -> tuple[list[SearchResult], int]: """Filter by relevance score and remove near-duplicate chunks (ARCH-056). Step 1: Remove chunks below min_score. - Step 2: If dedup_enabled, remove chunks with >80% token overlap with any + Step 2: If the threshold empties the set and rescue_top_k > 0, keep the + top rescue_top_k chunks scoring >= rescue_floor instead (TECH-007). + The threshold is a safety net, not a hard gate: multi-part + questions get uniformly low reranker scores (each chunk answers + only one part), so a borderline set reaches the LLM, which + arbitrates via grounding. + Step 3: If dedup_enabled, remove chunks with >80% token overlap with any already-selected chunk (keeping higher-scoring ones by processing in score-descending order). - Returns results in score-descending order. + Returns (filtered results, count of rescued chunks). The rescue and dedup + paths return results in score-descending order; the plain-threshold path + preserves the caller's ordering (already score-descending after search or + rerank). """ filtered = [r for r in results if r.score >= min_score] + rescued = 0 + if not filtered and rescue_top_k > 0: + filtered = sorted( + (r for r in results if r.score >= rescue_floor), + key=lambda r: r.score, + reverse=True, + )[:rescue_top_k] + rescued = len(filtered) + if not dedup_enabled: - return filtered + return filtered, rescued # Process by score descending; keep first occurrence of near-duplicates kept: list[SearchResult] = [] @@ -168,7 +188,7 @@ def _apply_retrieval_filter( if not is_dup: kept.append(candidate) - return kept + return kept, rescued # --------------------------------------------------------------------------- @@ -417,10 +437,12 @@ async def execute( # Step 3: Retrieval filter t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -431,6 +453,7 @@ async def execute( "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) @@ -733,10 +756,12 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] # Step 3: Retrieval filter t0 = time.monotonic() - filtered = _apply_retrieval_filter( + filtered, rescued = _apply_retrieval_filter( results, min_score=self._config.min_relevance_score, dedup_enabled=self._config.chunk_dedup_enabled, + rescue_top_k=self._config.retrieval_rescue_top_k, + rescue_floor=self._config.retrieval_rescue_floor, ) no_relevant_context = len(filtered) == 0 steps.append( @@ -747,6 +772,7 @@ async def _stream(self, query: QueryRequest) -> AsyncGenerator[QueryChunk, None] "before": len(results), "after": len(filtered), "no_relevant_context": no_relevant_context, + "rescued": rescued, }, ) ) diff --git a/vektra-core/src/vektra_core/reranker.py b/vektra-core/src/vektra_core/reranker.py index 87db0be4..eba02e74 100644 --- a/vektra-core/src/vektra_core/reranker.py +++ b/vektra-core/src/vektra_core/reranker.py @@ -14,7 +14,10 @@ import asyncio import dataclasses import math +from typing import Protocol, runtime_checkable +from urllib.parse import urlparse +import httpx import structlog from vektra_shared.config import RerankConfig @@ -30,6 +33,14 @@ } +def _redact_url(url: str) -> str: + """Return scheme://hostname:port only, stripping credentials and path.""" + parsed = urlparse(url) + host = parsed.hostname or "" + port = f":{parsed.port}" if parsed.port else "" + return f"{parsed.scheme}://{host}{port}" + + def _sigmoid(x: float) -> float: """Numerically stable sigmoid for cross-encoder logits.""" if x >= 0: @@ -49,6 +60,54 @@ class RerankResult: ] # (chunk_id, reranker_score, original_score) +@runtime_checkable +class RerankerProtocol(Protocol): + """Common interface of the in-process and remote reranker services.""" + + async def rerank( + self, + query: str, + results: list[SearchResult], + top_k: int, + ) -> RerankResult: ... + + +def _build_rerank_result( + results: list[SearchResult], + ordered: list[tuple[int, float]], + top_k: int, +) -> RerankResult: + """Build a RerankResult from (candidate_index, raw_score) pairs. + + Detects whether normalization is needed: FlashRank and TEI (with + raw_scores=false) produce sigmoid scores in [0, 1]; cross-encoder + logits can be negative or > 1. + """ + all_raw = [score for _, score in ordered] + needs_sigmoid = any(s < 0.0 or s > 1.0 for s in all_raw) + + all_scores: list[tuple[str, float, float]] = [] + reranked: list[SearchResult] = [] + + for idx, raw in ordered: + original = results[idx] + normalized = _sigmoid(raw) if needs_sigmoid else raw + all_scores.append( + (original.chunk_id, round(normalized, 4), round(original.score, 4)) + ) + + if len(reranked) < top_k: + reranked.append( + dataclasses.replace( + original, + score=normalized, + original_score=original.score, + ) + ) + + return RerankResult(top_k=reranked, all_scores=all_scores) + + class RerankerService: """Wraps the rerankers library for scoring and reordering search results.""" @@ -77,49 +136,89 @@ async def rerank( docs=docs, ) - # Score ALL candidates and detect whether normalization is needed. - # FlashRank produces sigmoid scores in [0, 1]; cross-encoder - # produces raw logits that can be negative or > 1. - all_items = ranked.results - all_raw = [float(item.score) for item in all_items] - needs_sigmoid = any(s < 0.0 or s > 1.0 for s in all_raw) - - all_scores: list[tuple[str, float, float]] = [] - reranked: list[SearchResult] = [] - - for item in all_items: - original = results[item.doc_id] - raw = float(item.score) - normalized = _sigmoid(raw) if needs_sigmoid else raw - all_scores.append( - (original.chunk_id, round(normalized, 4), round(original.score, 4)) - ) + ordered = [(item.doc_id, float(item.score)) for item in ranked.results] + return _build_rerank_result(results, ordered, top_k) - if len(reranked) < top_k: - reranked.append( - dataclasses.replace( - original, - score=normalized, - original_score=original.score, - ) - ) - return RerankResult(top_k=reranked, all_scores=all_scores) +class TEIRerankerService: + """Reranker backed by a TEI /rerank endpoint (FEAT-024). + + One TEI instance serves one reranker model (e.g. bge-reranker-v2-m3). + POST /rerank {"query", "texts", "raw_scores": false} returns + [{"index", "score"}] sorted by score descending, scores in [0, 1]. + """ + def __init__( + self, + *, + url: str, + api_key: str | None = None, + timeout_s: float = 30.0, + _client: httpx.AsyncClient | None = None, + ) -> None: + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + self._client = _client or httpx.AsyncClient( + base_url=url.rstrip("/"), headers=headers, timeout=timeout_s + ) + + async def rerank( + self, + query: str, + results: list[SearchResult], + top_k: int, + ) -> RerankResult: + """Rerank search results via the remote TEI cross-encoder.""" + if not results: + return RerankResult(top_k=[], all_scores=[]) + + resp = await self._client.post( + "/rerank", + json={ + "query": query, + "texts": [r.text_snippet for r in results], + "raw_scores": False, + }, + ) + resp.raise_for_status() + ranked = resp.json() + + ordered = [(int(item["index"]), float(item["score"])) for item in ranked] + return _build_rerank_result(results, ordered, top_k) -def create_reranker(config: RerankConfig) -> RerankerService | None: - """Create a RerankerService from config. Returns None if unavailable.""" + async def aclose(self) -> None: + """Release the underlying HTTP client (tests and shutdown).""" + await self._client.aclose() + + +def create_reranker(config: RerankConfig) -> RerankerProtocol | None: + """Create a reranker service from config. Returns None if unavailable.""" if not config.enabled: log.info("reranker_disabled") return None + if config.provider == "tei": + try: + service = TEIRerankerService(url=config.tei_url, api_key=config.tei_api_key) + log.info("reranker_loaded", provider="tei", url=_redact_url(config.tei_url)) + except Exception as exc: + # Same graceful-degradation contract as the in-process path: + # a bad reranker config must not abort startup. + log.warning("reranker_init_failed", provider="tei", error=str(exc)) + return None + return service + model_type = _PROVIDER_TO_MODEL_TYPE.get(config.provider, config.provider) model_name = config.model or _default_model_for_provider(config.provider) try: from rerankers import Reranker # type: ignore[import-untyped] - ranker = Reranker(model_name, model_type=model_type, verbose=0) + # API-based providers (cohere) need the key passed through; + # without it the option was dead as wired (FEAT-024). + kwargs: dict[str, str] = {} + if config.api_key: + kwargs["api_key"] = config.api_key + ranker = Reranker(model_name, model_type=model_type, verbose=0, **kwargs) if ranker is None: log.warning( "reranker_init_failed", diff --git a/vektra-core/src/vektra_core/templates.py b/vektra-core/src/vektra_core/templates.py index e6db0304..21e2ede4 100644 --- a/vektra-core/src/vektra_core/templates.py +++ b/vektra-core/src/vektra_core/templates.py @@ -95,16 +95,19 @@ def render_system( namespace: str = "default", grounding_mode: str = "strict", has_context: bool = True, + citations_enabled: bool = False, ) -> str: - """Render system.j2 with namespace, grounding mode, and context flag.""" + """Render system.j2 with namespace, grounding mode, and context/citation flags.""" tmpl = self._env.get_template("system.j2") return tmpl.render( namespace=namespace, grounding_mode=grounding_mode, has_context=has_context, + citations_enabled=citations_enabled, ) def render_context(self, chunks: list[dict[str, Any]]) -> str: - """Render context.j2 with a list of chunk dicts containing 'text' (and optionally 'score').""" + """Render context.j2 with a list of chunk dicts containing 'text' and + optionally 'score' and 'title' (source title attribute, FEAT-021).""" tmpl = self._env.get_template("context.j2") return tmpl.render(chunks=chunks) diff --git a/vektra-core/src/vektra_core/templates/context.j2 b/vektra-core/src/vektra_core/templates/context.j2 index cde635de..7a0364dc 100644 --- a/vektra-core/src/vektra_core/templates/context.j2 +++ b/vektra-core/src/vektra_core/templates/context.j2 @@ -1,3 +1,3 @@ -{% for chunk in chunks %}{{ chunk.text | e }} +{% for chunk in chunks %}{{ chunk.text | e }} {% endfor %} diff --git a/vektra-core/src/vektra_core/templates/system.j2 b/vektra-core/src/vektra_core/templates/system.j2 index 5eaf1dee..ed43925e 100644 --- a/vektra-core/src/vektra_core/templates/system.j2 +++ b/vektra-core/src/vektra_core/templates/system.j2 @@ -30,8 +30,16 @@ enough information to answer. {% endif %} {% endif %} Rules: +{% if citations_enabled and has_context %} +1. Cite the sources you used with inline [id] markers matching the id + attributes of the elements, placed at the end of the sentence + they support. If multiple sources support a claim, list them together, + e.g. [1][3]. Do not quote source titles or talk about the retrieval + process; the markers are the only visible reference. +{% else %} 1. Sound like you simply know the answer. Never mention, quote, or allude to sources, documents, context tags, or reference material. +{% endif %} 2. Never offer to search, look up, or provide more information later. 3. If a source is cut off, use what is available without commenting on it. 4. Respond in the same language the user writes in. diff --git a/vektra-core/tests/conftest.py b/vektra-core/tests/conftest.py new file mode 100644 index 00000000..577e6fbb --- /dev/null +++ b/vektra-core/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared fixtures for vektra-core unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding config defaults (e.g. VEKTRA_PARENT_EXPANSION_ENABLED, + VEKTRA_EVAL_MODE). CI has no .env and never sees the difference. Tests + that need a specific value still set it explicitly via constructor + kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) diff --git a/vektra-core/tests/test_advanced_pipeline.py b/vektra-core/tests/test_advanced_pipeline.py index bdd1c488..83d4cd50 100644 --- a/vektra-core/tests/test_advanced_pipeline.py +++ b/vektra-core/tests/test_advanced_pipeline.py @@ -155,6 +155,35 @@ async def test_execute_no_relevant_context(): assert response.answer is None +async def test_execute_rescue_keeps_borderline_chunks(): + """With rescue enabled, a set wiped by the threshold reaches the LLM (TECH-007).""" + results = [ + _make_search_result(0.09, "partial answer one"), + _make_search_result(0.07, "partial answer two"), + _make_search_result(0.01, "below the rescue floor"), + ] + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=results) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config( + VEKTRA_RETRIEVAL_RESCUE_TOP_K=3, + VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.02, + ), + ) + response, trace = await pipeline.execute( + QueryRequest(question="multi-part question") + ) + + assert response.no_relevant_context is False + assert response.answer == "The answer." + assert len(response.sources) == 2 + filter_step = next(s for s in trace.steps if s.name == "retrieval_filter") + assert filter_step.metadata["rescued"] == 2 + assert filter_step.metadata["after"] == 2 + + async def test_execute_no_relevant_context_hybrid_calls_llm(): """In hybrid mode, LLM is called even when no chunks pass threshold.""" results = [_make_search_result(0.1, "irrelevant")] @@ -770,3 +799,262 @@ async def test_eval_mode_off_excludes_prompt_messages(): build_step = next(s for s in trace.steps if s.name == "build_prompt") assert "messages" not in build_step.metadata + + +# --------------------------------------------------------------------------- +# Parent chunk expansion (FEAT-017) +# --------------------------------------------------------------------------- + + +def _make_child_result( + score: float, text: str, parent_id: str, doc_id: UUID | None = None +) -> SearchResult: + return SearchResult( + chunk_id=str(uuid4()), + score=score, + text_snippet=text, + document_id=doc_id or uuid4(), + document_version=1, + parent_id=parent_id, + ) + + +def _make_parent_chunk(chunk_id: str, text: str, doc_id: UUID) -> SearchResult: + return SearchResult( + chunk_id=chunk_id, + score=0.0, + text_snippet=text, + document_id=doc_id, + document_version=1, + ) + + +async def test_parent_expansion_replaces_child_text(): + """Expanded children carry the parent text into prompt and sources.""" + doc_id = uuid4() + parent_text = "PARENT SECTION: full surrounding context for the child." + child = _make_child_result(0.9, "tiny child snippet", "parent-1", doc_id) + orphan = _make_search_result(0.8, "standalone chunk without parent") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child, orphan]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute( + QueryRequest(question="test", namespace="default") + ) + + vector_store.retrieve.assert_awaited_once_with( + namespace="default", chunk_ids=["parent-1"] + ) + # Expanded result keeps the child's chunk_id and score + by_id = {s.chunk_id: s for s in response.sources} + assert by_id[child.chunk_id].snippet == parent_text + assert by_id[child.chunk_id].score == child.score + assert by_id[orphan.chunk_id].snippet == orphan.text_snippet + + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 1 + assert step.metadata["siblings_merged"] == 0 + assert step.metadata["parents_fetched"] == 1 + + +async def test_parent_expansion_merges_siblings_of_same_parent(): + """Children of the same parent collapse into the highest-scored one.""" + doc_id = uuid4() + parent_text = "PARENT: contains both sibling fragments." + sibling_hi = _make_child_result(0.9, "fragment one", "parent-1", doc_id) + sibling_lo = _make_child_result(0.7, "fragment two", "parent-1", doc_id) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[sibling_hi, sibling_lo]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert len(response.sources) == 1 + assert response.sources[0].chunk_id == sibling_hi.chunk_id + assert response.sources[0].snippet == parent_text + + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 1 + assert step.metadata["siblings_merged"] == 1 + + +async def test_parent_expansion_disabled_by_default(): + """Without the flag no retrieve call is made and no step is traced.""" + child = _make_child_result(0.9, "child snippet", "parent-1") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + + pipeline = _make_pipeline(vector_store=vector_store) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + vector_store.retrieve.assert_not_awaited() + assert all(s.name != "parent_expansion" for s in trace.steps) + assert response.sources[0].snippet == "child snippet" + + +async def test_parent_expansion_retrieve_failure_keeps_children(): + """A vector store failure during expansion degrades gracefully.""" + child = _make_child_result(0.9, "child snippet", "parent-1") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock(side_effect=RuntimeError("qdrant down")) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert response.answer is not None + assert response.sources[0].snippet == "child snippet" + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata.get("skipped") is True + + +async def test_parent_expansion_missing_parent_keeps_child(): + """Children whose parent is not returned by retrieve() stay unexpanded.""" + child = _make_child_result(0.9, "child snippet", "parent-gone") + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock(return_value=[]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + assert response.sources[0].snippet == "child snippet" + step = next(s for s in trace.steps if s.name == "parent_expansion") + assert step.metadata["children_expanded"] == 0 + assert step.metadata["parents_fetched"] == 0 + + +async def test_parent_expansion_feeds_token_budget_with_parent_text(): + """Token budgeting (ARCH-055) operates on the expanded parent text.""" + doc_id = uuid4() + parent_text = "PARENT TEXT " * 50 + child = _make_child_result(0.9, "tiny", "parent-1", doc_id) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[child]) + vector_store.retrieve = AsyncMock( + return_value=[_make_parent_chunk("parent-1", parent_text, doc_id)] + ) + + llm = MagicMock() + llm.complete = AsyncMock( + return_value=CompletionResponse( + content="The answer.", + model="ollama/llama3", + prompt_tokens=10, + completion_tokens=20, + total_tokens=30, + ) + ) + token_counts: list[str] = [] + + def _count(text: str, model: str | None = None) -> int: + token_counts.append(text) + return len(text.split()) + + llm.count_tokens = MagicMock(side_effect=_count) + + pipeline = _make_pipeline( + vector_store=vector_store, + llm=llm, + pipeline_config=_make_pipeline_config(VEKTRA_PARENT_EXPANSION_ENABLED=True), + ) + await pipeline.execute(QueryRequest(question="test")) + + # The budget counted the parent text, not the child snippet + assert any(parent_text == t for t in token_counts) + assert all(t != "tiny" for t in token_counts) + + +# --------------------------------------------------------------------------- +# Per-namespace citations (FEAT-021) +# --------------------------------------------------------------------------- + + +async def test_citations_enabled_flows_to_prompt_and_sources(): + """citations_enabled=True: Rule 1 swaps, context gets titles, SourceRef.title set.""" + from unittest.mock import patch + from uuid import UUID as _UUID + + doc_id = uuid4() + result = SearchResult( + chunk_id=str(uuid4()), + score=0.9, + text_snippet="Article 21 grants freedom of expression.", + document_id=doc_id, + document_version=1, + metadata={"page": 12}, + ) + + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[result]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + + with patch( + "vektra_core.advanced_pipeline._fetch_document_names", + new=AsyncMock(return_value={str(doc_id): "lecture-07.pdf"}), + ): + response, trace = await pipeline.execute( + QueryRequest(question="test", citations_enabled=True) + ) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + messages = build_step.metadata["messages"] + system_msg = next(m for m in messages if m["role"] == "system") + user_msg = next(m for m in messages if m["role"] == "user") + assert "Cite the sources" in system_msg["content"] + assert 'title="lecture-07.pdf, p.12"' in user_msg["content"] + + assert response.sources[0].title == "lecture-07.pdf, p.12" + assert response.sources[0].document_name == "lecture-07.pdf" + assert isinstance(response.sources[0].doc_id, _UUID) + + +async def test_citations_disabled_by_default_leaves_prompt_untouched(): + """Default QueryRequest: hidden-sources rule, no titles, SourceRef.title None.""" + result = _make_search_result(0.9, "some grounded text") + vector_store = AsyncMock() + vector_store.search = AsyncMock(return_value=[result]) + + pipeline = _make_pipeline( + vector_store=vector_store, + pipeline_config=_make_pipeline_config(VEKTRA_EVAL_MODE=True), + ) + response, trace = await pipeline.execute(QueryRequest(question="test")) + + build_step = next(s for s in trace.steps if s.name == "build_prompt") + messages = build_step.metadata["messages"] + system_msg = next(m for m in messages if m["role"] == "system") + user_msg = next(m for m in messages if m["role"] == "user") + assert "Never mention, quote, or allude" in system_msg["content"] + assert "Cite the sources" not in system_msg["content"] + assert "title=" not in user_msg["content"] + assert response.sources[0].title is None diff --git a/vektra-core/tests/test_api.py b/vektra-core/tests/test_api.py index 59d49613..56c15b38 100644 --- a/vektra-core/tests/test_api.py +++ b/vektra-core/tests/test_api.py @@ -60,6 +60,7 @@ async def _execute(query_req): score=0.9, snippet="Some relevant snippet.", citation_id=uuid4(), + title="lecture-07.pdf, p.3", # FEAT-021 ) ], conversation_id=query_req.conversation_id, @@ -132,6 +133,8 @@ async def test_query_json_returns_response(): assert isinstance(body["sources"], list) assert len(body["sources"]) == 1 assert "citation_id" in body["sources"][0] + # FEAT-021: the title must survive the SourceRef -> SourceRefBody mapping + assert body["sources"][0]["title"] == "lecture-07.pdf, p.3" async def test_query_requires_auth(): diff --git a/vektra-core/tests/test_pipeline.py b/vektra-core/tests/test_pipeline.py index ba03600c..db3a6e05 100644 --- a/vektra-core/tests/test_pipeline.py +++ b/vektra-core/tests/test_pipeline.py @@ -135,10 +135,13 @@ def test_retrieval_filter_removes_low_score(): _make_search_result(0.2, "bad chunk"), _make_search_result(0.8, "great chunk"), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) + filtered, rescued = _apply_retrieval_filter( + results, min_score=0.3, dedup_enabled=False + ) scores = [r.score for r in filtered] assert all(s >= 0.3 for s in scores) assert len(filtered) == 2 + assert rescued == 0 def test_retrieval_filter_dedup_removes_near_duplicate(): @@ -148,7 +151,7 @@ def test_retrieval_filter_dedup_removes_near_duplicate(): _make_search_result(0.9, text_a), _make_search_result(0.7, text_b), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=True) + filtered, _ = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=True) assert len(filtered) == 1 assert filtered[0].score == 0.9 # higher-scoring one kept @@ -160,10 +163,90 @@ def test_retrieval_filter_no_dedup_keeps_all(): _make_search_result(0.9, text_a), _make_search_result(0.7, text_b), ] - filtered = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) + filtered, _ = _apply_retrieval_filter(results, min_score=0.3, dedup_enabled=False) assert len(filtered) == 2 +def test_retrieval_filter_rescue_disabled_by_default(): + results = [ + _make_search_result(0.09, "part one"), + _make_search_result(0.07, "part two"), + ] + filtered, rescued = _apply_retrieval_filter( + results, min_score=0.15, dedup_enabled=False + ) + assert filtered == [] + assert rescued == 0 + + +def test_retrieval_filter_rescue_keeps_top_k_above_floor(): + results = [ + _make_search_result(0.004, "below floor"), + _make_search_result(0.09, "part one"), + _make_search_result(0.05, "part three"), + _make_search_result(0.07, "part two"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert [r.score for r in filtered] == [0.09, 0.07, 0.05] + assert rescued == 3 + + +def test_retrieval_filter_rescue_all_below_floor_returns_empty(): + results = [ + _make_search_result(0.01, "noise"), + _make_search_result(0.005, "more noise"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert filtered == [] + assert rescued == 0 + + +def test_retrieval_filter_rescue_not_triggered_when_survivors_exist(): + results = [ + _make_search_result(0.5, "strong chunk"), + _make_search_result(0.09, "weak chunk"), + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=False, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert [r.score for r in filtered] == [0.5] + assert rescued == 0 + + +def test_retrieval_filter_rescued_chunks_pass_through_dedup(): + text = "the quick brown fox jumps over the lazy dog and more words here" + results = [ + _make_search_result(0.09, text), + _make_search_result(0.07, text + " again"), # >80% overlap with the first + ] + filtered, rescued = _apply_retrieval_filter( + results, + min_score=0.15, + dedup_enabled=True, + rescue_top_k=3, + rescue_floor=0.02, + ) + assert len(filtered) == 1 + assert filtered[0].score == 0.09 + assert rescued == 2 # rescued before dedup + + # --------------------------------------------------------------------------- # Pipeline execute() tests # --------------------------------------------------------------------------- diff --git a/vektra-core/tests/test_reranker.py b/vektra-core/tests/test_reranker.py index ca631da4..9621653a 100644 --- a/vektra-core/tests/test_reranker.py +++ b/vektra-core/tests/test_reranker.py @@ -1,12 +1,17 @@ -"""Unit tests for RerankerService and create_reranker factory.""" +"""Unit tests for RerankerService, TEIRerankerService and create_reranker.""" +import json from types import SimpleNamespace from unittest.mock import MagicMock, patch from uuid import uuid4 +import httpx +import pytest + from vektra_core.reranker import ( RerankerService, RerankResult, + TEIRerankerService, _default_model_for_provider, _sigmoid, create_reranker, @@ -242,3 +247,83 @@ def test_sigmoid_extreme_values_no_overflow(): """Numerically stable sigmoid must not overflow on extreme logits.""" assert _sigmoid(1000.0) == 1.0 assert _sigmoid(-1000.0) == 0.0 + + +# --------------------------------------------------------------------------- +# TEIRerankerService (FEAT-024) +# --------------------------------------------------------------------------- + + +def _tei_client(handler) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + + +async def test_tei_rerank_empty_results(): + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(None)) + result = await service.rerank("query", [], top_k=5) + assert result.top_k == [] + assert result.all_scores == [] + + +async def test_tei_rerank_orders_and_normalizes(): + results = [ + _make_result(0.5, "doc A"), + _make_result(0.3, "doc B"), + _make_result(0.9, "doc C"), + ] + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert request.url.path == "/rerank" + assert body["query"] == "test query" + assert body["texts"] == ["doc A", "doc B", "doc C"] + assert body["raw_scores"] is False + return httpx.Response( + 200, + json=[ + {"index": 2, "score": 0.95}, + {"index": 0, "score": 0.80}, + {"index": 1, "score": 0.10}, + ], + ) + + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(handler)) + result = await service.rerank("test query", results, top_k=2) + + assert len(result.top_k) == 2 + assert result.top_k[0].chunk_id == results[2].chunk_id + assert result.top_k[0].score == 0.95 + assert result.top_k[0].original_score == 0.9 # BUG-015 preserved + assert result.top_k[1].chunk_id == results[0].chunk_id + assert len(result.all_scores) == 3 + + +async def test_tei_rerank_raises_on_http_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + service = TEIRerankerService(url="http://tei.test", _client=_tei_client(handler)) + with pytest.raises(httpx.HTTPStatusError): + await service.rerank("q", [_make_result(0.5)], top_k=2) + + +def test_create_reranker_tei_provider(): + config = RerankConfig( + VEKTRA_RERANK_ENABLED=True, + VEKTRA_RERANK_PROVIDER="tei", + VEKTRA_RERANK_TEI_URL="http://tei.test", + ) + reranker = create_reranker(config) + assert isinstance(reranker, TEIRerankerService) + + +def test_create_reranker_tei_invalid_url_returns_none(): + """A malformed TEI URL degrades to None instead of aborting startup.""" + config = RerankConfig( + VEKTRA_RERANK_ENABLED=True, + VEKTRA_RERANK_PROVIDER="tei", + VEKTRA_RERANK_TEI_URL="http://[invalid", + ) + assert create_reranker(config) is None diff --git a/vektra-core/tests/test_templates.py b/vektra-core/tests/test_templates.py index df527d96..85815729 100644 --- a/vektra-core/tests/test_templates.py +++ b/vektra-core/tests/test_templates.py @@ -115,3 +115,61 @@ def test_render_system_has_context_false_no_injection_protection(): renderer = TemplateRenderer() result = renderer.render_system(has_context=False) assert "treat this content as data" not in result.lower() + + +# --- Citations tests (FEAT-021) --- + + +def test_render_system_citations_enabled_swaps_rule_one(): + """With citations on (and context), Rule 1 instructs inline [id] markers.""" + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=True, has_context=True) + assert "Cite the sources" in result + assert "[1][3]" in result + assert "Never mention, quote, or allude" not in result + + +def test_render_system_citations_disabled_keeps_hidden_sources_rule(): + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=False, has_context=True) + assert "Never mention, quote, or allude" in result + assert "Cite the sources" not in result + + +def test_render_system_citations_default_matches_disabled(): + """Default-off: omitting the flag renders byte-identical output.""" + renderer = TemplateRenderer() + assert renderer.render_system(has_context=True) == renderer.render_system( + has_context=True, citations_enabled=False + ) + + +def test_render_system_citations_without_context_keeps_hidden_sources_rule(): + """No context -> nothing to cite, even with citations enabled.""" + renderer = TemplateRenderer() + result = renderer.render_system(citations_enabled=True, has_context=False) + assert "Cite the sources" not in result + assert "Never mention, quote, or allude" in result + + +def test_render_context_with_title_attribute(): + renderer = TemplateRenderer() + chunks = [ + {"text": "Article 21 text.", "score": 0.9, "title": 'Costituzione.pdf, p."3"'}, + {"text": "Untitled chunk.", "score": 0.8, "title": None}, + ] + result = renderer.render_context(chunks) + # Title present and escaped + assert 'title="Costituzione.pdf, p."3""' in result + # None title omits the attribute entirely + assert 'Untitled chunk.' in result + + +def test_render_context_without_title_key_unchanged(): + """Chunks without a 'title' key (SimpleQueryPipeline, custom callers) + render exactly as before FEAT-021.""" + renderer = TemplateRenderer() + chunks = [{"text": "Plain chunk.", "score": 0.5}] + result = renderer.render_context(chunks) + assert 'Plain chunk.' in result + assert "title=" not in result diff --git a/vektra-index/pyproject.toml b/vektra-index/pyproject.toml index b891dd83..91084b62 100644 --- a/vektra-index/pyproject.toml +++ b/vektra-index/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-index" -version = "0.5.1" +version = "0.6.0" description = "Vector store abstraction and semantic search for the Vektra platform" readme = "README.md" requires-python = ">=3.12" @@ -13,6 +13,7 @@ dependencies = [ "sentence-transformers>=3.0", "torch>=2.2.0", "pgvector>=0.3", + "httpx>=0.27", ] [project.optional-dependencies] diff --git a/vektra-index/src/vektra_index/adapters.py b/vektra-index/src/vektra_index/adapters.py index 4b4d184d..efb67280 100644 --- a/vektra-index/src/vektra_index/adapters.py +++ b/vektra-index/src/vektra_index/adapters.py @@ -120,6 +120,17 @@ async def search( raw_filters=raw_filters, ) + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + factory = self._get_session_factory() + pgvector = self._get_pgvector() + + async with factory() as session: + return await pgvector.retrieve(session, namespace, chunk_ids) + async def delete(self, namespace: str, ids: list[str]) -> int: """Delete all chunks for each document_id in ids. diff --git a/vektra-index/src/vektra_index/api.py b/vektra-index/src/vektra_index/api.py index 5cf76738..3fff16b6 100644 --- a/vektra-index/src/vektra_index/api.py +++ b/vektra-index/src/vektra_index/api.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from vektra_shared.auth import ApiKeyInfo, require_scope -from vektra_shared.config import EmbeddingConfig, VectorStoreConfig +from vektra_shared.config import VectorStoreConfig from vektra_shared.db import get_session from vektra_shared.errors import ( ERR_INGEST_004, @@ -37,7 +37,6 @@ # Read from env once at import time (immutable for process lifetime) _VS_CONFIG = VectorStoreConfig() -_EMB_CONFIG = EmbeddingConfig() router = APIRouter(prefix="/api/v1", tags=["index"]) @@ -199,31 +198,26 @@ async def search( request: Request, body: SearchRequest, key: ApiKeyInfo = Depends(require_scope("query")), - session: AsyncSession = Depends(get_session), ) -> SearchResponse: """Semantic search over indexed chunks (no LLM synthesis). - Embeds the query string, then runs cosine similarity search with optional - JSONB metadata filtering. Returns ranked chunks. + Embeds the query string, then runs similarity search on the active + vector store provider with optional JSONB metadata filtering. + Returns ranked chunks. """ import logging - from vektra_index.providers.pgvector import PgvectorProvider - from vektra_index.providers.sentence_transformers import ( - SentenceTransformersProvider, - ) - _logger = logging.getLogger(__name__) + # Providers come from the registry: search must hit the active vector + # store (Qdrant when configured), not a hardcoded pgvector (BUG-021). + registry = request.app.state.registry + # Enforce namespace binding for scoped keys (H5) effective_ns = key.namespace_id or body.namespace - embedding_provider = SentenceTransformersProvider( - model_name=_EMB_CONFIG.embedding_model - ) - pgvector_provider = PgvectorProvider( - active_index_version=_VS_CONFIG.active_index_version - ) + embedding_provider = registry.get("embedding", "default") + vector_store = registry.get("vector_store", "default") # Embed the query (dense) - skip for SPARSE-only mode (NP24) dense_vector: list[float] = [] @@ -242,8 +236,8 @@ async def search( sparse_vector = None if body.search_mode in (SearchMode.SPARSE, SearchMode.HYBRID): - sparse_provider = getattr(request.app.state, "sparse_embedding_provider", None) - if sparse_provider is not None: + if registry.has("sparse_embedding", "default"): + sparse_provider = registry.get("sparse_embedding", "default") try: sparse_vector = await sparse_provider.embed_query(body.query) except Exception as exc: @@ -275,8 +269,7 @@ async def search( filters = body.filters # type: ignore[assignment] try: - results = await pgvector_provider.search( - session=session, + results = await vector_store.search( namespace=effective_ns, query_embedding=query_embedding, top_k=body.top_k, @@ -288,7 +281,10 @@ async def search( category=ErrorCategory.UPSTREAM, code=ERR_QUERY_004, message=f"Vector store read failed: {exc}", - remediation="Check PostgreSQL connectivity and pgvector extension status.", + remediation=( + "Check vector store connectivity and status. " + "Run GET /health for component status." + ), ) raise HTTPException( status_code=http_status_for(err), detail=err.to_envelope() diff --git a/vektra-index/src/vektra_index/providers/pgvector.py b/vektra-index/src/vektra_index/providers/pgvector.py index 62f4cb03..7834fa83 100644 --- a/vektra-index/src/vektra_index/providers/pgvector.py +++ b/vektra-index/src/vektra_index/providers/pgvector.py @@ -73,7 +73,18 @@ async def store( inserted_ids: list[str] = [] for position, chunk in enumerate(chunks): - chunk_id = uuid4() + # Honor caller-provided deterministic ids (uuid5 from ingest); + # parent linkage relies on them (FEAT-017). + try: + chunk_id = UUID(chunk.chunk_id) if chunk.chunk_id else uuid4() + except ValueError: + chunk_id = uuid4() + parent_uuid: UUID | None = None + if chunk.parent_id: + try: + parent_uuid = UUID(chunk.parent_id) + except ValueError: + parent_uuid = None sparse_data = None if chunk.sparse is not None: sparse_data = { @@ -90,6 +101,7 @@ async def store( chunk_metadata=chunk.metadata, position=chunk.metadata.get("position", position), index_version=self._active_index_version, + parent_id=parent_uuid, ) session.add(orm_obj) inserted_ids.append(str(chunk_id)) @@ -151,6 +163,7 @@ async def _search_dense( DocumentChunkOrm.document_id, DocumentChunkOrm.content, DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, score_expr, ) .where( @@ -161,6 +174,7 @@ async def _search_dense( .limit(top_k) ) + stmt = self._exclude_parent_chunks(stmt) stmt = self._apply_filters(stmt, filters) stmt = self._join_source_documents(stmt) @@ -228,6 +242,7 @@ async def _search_sparse( DocumentChunkOrm.document_id, DocumentChunkOrm.content, DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, score_col, ) .where( @@ -239,6 +254,7 @@ async def _search_sparse( .limit(top_k) ) + stmt = self._exclude_parent_chunks(stmt) stmt = self._apply_filters(stmt, filters) stmt = self._join_source_documents(stmt) @@ -314,10 +330,27 @@ async def _search_hybrid( document_id=r.document_id, document_version=r.document_version, metadata=r.metadata, + parent_id=r.parent_id, ) for rrf_score, r in rrf_scored[:top_k] ] + @staticmethod + def _exclude_parent_chunks(stmt: Any) -> Any: + """Exclude parent-level chunks from search results (FEAT-017). + + Parent chunks are context material fetched by id during expansion, + not retrieval targets. Chunks without a chunk_level key (fixed + chunking) are unaffected: NULL IS DISTINCT FROM 'parent'. + """ + from vektra_index.models import DocumentChunkOrm + + return stmt.where( + DocumentChunkOrm.chunk_metadata["chunk_level"] + .as_string() + .is_distinct_from("parent") + ) + @staticmethod def _apply_filters(stmt: Any, filters: SearchFilters | None) -> Any: """Apply JSONB metadata filters in the same SQL query (REQ-063).""" @@ -361,10 +394,48 @@ def _rows_to_results(rows: Sequence[Any]) -> list[SearchResult]: document_id=row.document_id, document_version=row.document_version, metadata=row.chunk_metadata or {}, + parent_id=str(row.parent_id) if row.parent_id else None, ) for row in rows ] + async def retrieve( + self, + session: AsyncSession, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + """Fetch chunks by id (no vector search). Used for parent chunk + expansion (FEAT-017); score is 0.0 by convention. + """ + from vektra_index.models import DocumentChunkOrm + + valid_ids: list[UUID] = [] + for cid in chunk_ids: + try: + valid_ids.append(UUID(cid)) + except ValueError: + logger.warning("retrieve_invalid_chunk_id: %s", cid) + if not valid_ids: + return [] + + stmt = select( + DocumentChunkOrm.id, + DocumentChunkOrm.document_id, + DocumentChunkOrm.content, + DocumentChunkOrm.chunk_metadata, + DocumentChunkOrm.parent_id, + literal_column("0.0").label("score"), + ).where( + DocumentChunkOrm.id.in_(valid_ids), + DocumentChunkOrm.namespace_id == namespace, + DocumentChunkOrm.index_version == self._active_index_version, + ) + stmt = self._join_source_documents(stmt) + + result = await session.execute(stmt) + return self._rows_to_results(result.all()) + async def delete( self, session: AsyncSession, diff --git a/vektra-index/src/vektra_index/providers/qdrant.py b/vektra-index/src/vektra_index/providers/qdrant.py index bb27d8ae..9e073c16 100644 --- a/vektra-index/src/vektra_index/providers/qdrant.py +++ b/vektra-index/src/vektra_index/providers/qdrant.py @@ -106,13 +106,33 @@ async def ensure_collection(self) -> None: """Create the collection if it doesn't exist. Called during startup validation. Configures named vectors - for dense and sparse search. + for dense and sparse search. If the collection already exists, + verifies its dense vector size matches the active embedding model + (FEAT-024): a silent mismatch would fail on every upsert/search + with an opaque Qdrant error, so fail fast with a clear message. """ from qdrant_client import models collections = await self._client.get_collections() existing = {c.name for c in collections.collections} if self._collection_name in existing: + info = await self._client.get_collection(self._collection_name) + vectors = info.config.params.vectors + dense = ( + vectors.get("dense") + if isinstance(vectors, dict) + else getattr(vectors, "dense", None) + ) + existing_size = getattr(dense, "size", None) + if existing_size is not None and existing_size != self._dense_dimensions: + raise ValueError( + f"Qdrant collection '{self._collection_name}' has dense " + f"vectors of size {existing_size}, but the active embedding " + f"model produces {self._dense_dimensions} dimensions. " + "Changing the embedding model requires re-ingesting into a " + "new collection (set VEKTRA_QDRANT_COLLECTION) or deleting " + "the existing one." + ) return try: @@ -184,6 +204,7 @@ async def store( "text": chunk.text, "metadata": chunk.metadata, "document_id": chunk.metadata.get("document_id", ""), + "parent_id": chunk.parent_id, }, ) ) @@ -333,6 +354,38 @@ async def _search_hybrid( ) return self._points_to_results(results.points) + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: + """Fetch points by id (no vector search). Used for parent chunk + expansion (FEAT-017); score is 0.0 by convention. + + Points whose payload namespace does not match are dropped (namespace + isolation): Qdrant retrieve() takes no filter. + """ + valid_ids: list[str] = [] + for cid in chunk_ids: + try: + UUID(cid) + except ValueError: + logger.warning("qdrant_retrieve_invalid_chunk_id: %s", cid) + continue + valid_ids.append(cid) + if not valid_ids: + return [] + + records = await self._client.retrieve( + collection_name=self._collection_name, + ids=valid_ids, + with_payload=True, + ) + matching = [ + r for r in records if (r.payload or {}).get("namespace_id") == namespace + ] + return self._points_to_results(matching) + async def delete(self, namespace: str, ids: list[str]) -> int: """Delete points by document_id payload filter. @@ -428,11 +481,24 @@ def _build_filter( ) ) - return models.Filter(must=must_conditions) + # Parent chunks are context material, not retrieval targets (FEAT-017) + return models.Filter( + must=must_conditions, + must_not=[ + models.FieldCondition( + key="metadata.chunk_level", + match=models.MatchValue(value="parent"), + ) + ], + ) @staticmethod def _points_to_results(points: list[Any]) -> list[SearchResult]: - """Convert Qdrant ScoredPoint list to SearchResult list.""" + """Convert Qdrant ScoredPoint/Record list to SearchResult list. + + Record objects (from retrieve()) have no score attribute: score + defaults to 0.0. + """ results: list[SearchResult] = [] for point in points: payload = point.payload or {} @@ -442,16 +508,18 @@ def _points_to_results(points: list[Any]) -> list[SearchResult]: except (ValueError, AttributeError): doc_id = UUID(int=0) + score = getattr(point, "score", None) results.append( SearchResult( chunk_id=str(point.id), - score=float(point.score) if point.score is not None else 0.0, + score=float(score) if score is not None else 0.0, text_snippet=payload.get("text", ""), document_id=doc_id, document_version=payload.get("metadata", {}).get( "document_version", 1 ), metadata=payload.get("metadata", {}), + parent_id=payload.get("parent_id"), ) ) return results diff --git a/vektra-index/src/vektra_index/providers/tei.py b/vektra-index/src/vektra_index/providers/tei.py new file mode 100644 index 00000000..ffead977 --- /dev/null +++ b/vektra-index/src/vektra_index/providers/tei.py @@ -0,0 +1,125 @@ +"""TEIEmbeddingProvider: EmbeddingProvider over HuggingFace Text Embeddings Inference. + +Remote embedding via a TEI instance (FEAT-024, ADR-0013). Lets deployments +reuse a shared inference service (e.g. bge-m3 on the host) instead of +duplicating in-process CPU embedding per container, and unlocks models +whose sequence window exceeds the in-process default (bge-m3: 8192 tokens +vs 128 for paraphrase-multilingual-MiniLM-L12-v2). + +Endpoints used (TEI native API): +- POST /embed {"inputs": [...]} -> [[...], ...] +- GET /info for the embedding size (with an /embed probe fallback) + +Auth: optional Bearer token (TEI --api-key). +""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +import httpx + +from vektra_shared.types import HealthStatus + +logger = logging.getLogger(__name__) + +# TEI rejects batches larger than its --max-client-batch-size (default 32). +_MAX_BATCH = 32 + + +class TEIEmbeddingProvider: + """EmbeddingProvider backed by a remote TEI instance. + + Symmetric encoding: TEI applies the model's own pooling/normalization; + query and document paths use the same endpoint. + """ + + def __init__( + self, + *, + url: str, + api_key: str | None = None, + timeout_s: float = 30.0, + _client: httpx.AsyncClient | None = None, + ) -> None: + self._url = url.rstrip("/") + self._api_key = api_key + self._timeout_s = timeout_s + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + self._client = _client or httpx.AsyncClient( + base_url=self._url, headers=headers, timeout=timeout_s + ) + self._dimensions: int | None = None + + async def _embed_batch(self, texts: list[str]) -> list[list[float]]: + resp = await self._client.post("/embed", json={"inputs": texts}) + resp.raise_for_status() + data: list[list[float]] = resp.json() + if data and self._dimensions is None: + # Warm the dimensions cache so the startup warmup (embed_query) + # makes the later synchronous dimensions() call free. + self._dimensions = len(data[0]) + return data + + async def embed_documents(self, texts: list[str]) -> list[list[float]]: + """Embed a batch of document passages, chunked to the TEI batch limit.""" + out: list[list[float]] = [] + for i in range(0, len(texts), _MAX_BATCH): + out.extend(await self._embed_batch(texts[i : i + _MAX_BATCH])) + return out + + async def embed_query(self, query: str) -> list[float]: + """Embed a single query string for retrieval.""" + return (await self._embed_batch([query]))[0] + + def dimensions(self) -> int: + """Return the embedding dimensionality, fetched once from the server. + + The Protocol method is synchronous, so this uses a one-off sync HTTP + call (startup/wiring path, not the query hot path). Tries /info + first; TEI versions that do not expose the size fall back to probing + /embed with a single input. + """ + if self._dimensions is not None: + return self._dimensions + + headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {} + with httpx.Client( + base_url=self._url, headers=headers, timeout=self._timeout_s + ) as client: + self._dimensions = self._fetch_dimensions(client) + + logger.info("TEI embedding provider: %s (%d dims)", self._url, self._dimensions) + return self._dimensions + + @staticmethod + def _fetch_dimensions(client: httpx.Client) -> int: + try: + resp = client.get("/info") + resp.raise_for_status() + info: Any = resp.json() + if isinstance(info, dict): + for key in ("embedding_size", "hidden_size"): + if isinstance(info.get(key), int): + return int(info[key]) + except (httpx.HTTPError, ValueError): + logger.debug("TEI /info unavailable or invalid, probing /embed") + resp = client.post("/embed", json={"inputs": ["dim probe"]}) + resp.raise_for_status() + return len(resp.json()[0]) + + async def health_check(self) -> HealthStatus: + """Verify the TEI server responds and can produce an embedding.""" + try: + start = time.monotonic() + await self.embed_query("health check") + latency_ms = int((time.monotonic() - start) * 1000) + return HealthStatus(status="healthy", latency_ms=latency_ms) + except Exception as exc: + return HealthStatus(status="unhealthy", message=str(exc)) + + async def aclose(self) -> None: + """Release the underlying HTTP client (tests and shutdown).""" + await self._client.aclose() diff --git a/vektra-index/src/vektra_index/startup.py b/vektra-index/src/vektra_index/startup.py index 1ffd8575..ab4a6d2c 100644 --- a/vektra-index/src/vektra_index/startup.py +++ b/vektra-index/src/vektra_index/startup.py @@ -52,7 +52,9 @@ async def check_provider_registration( async def check_embedding_model(registry: Any) -> None: """ARCH-057 step 6: warm up the embedding model and verify dimensionality.""" try: - embedding_provider = registry.get("embedding", "sentence-transformers") + # "default" aliases whichever provider is active + # (sentence-transformers or tei, FEAT-024). + embedding_provider = registry.get("embedding", "default") test_embedding = await embedding_provider.embed_query("startup validation test") actual_dims = len(test_embedding) expected_dims = embedding_provider.dimensions() diff --git a/vektra-index/tests/test_api_search.py b/vektra-index/tests/test_api_search.py new file mode 100644 index 00000000..f494ac40 --- /dev/null +++ b/vektra-index/tests/test_api_search.py @@ -0,0 +1,157 @@ +"""Unit tests for /api/v1/search provider resolution (BUG-021). + +The endpoint must resolve embedding, sparse embedding, and vector store from +the ProviderRegistry (same contract as the query pipeline), so that Qdrant +deployments search the active store instead of a hardcoded pgvector instance. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from vektra_shared.auth import ApiKeyInfo +from vektra_shared.registry import ProviderRegistry +from vektra_shared.types import SearchMode, SearchResult + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_key_store(scopes: list[str]) -> AsyncMock: + """Minimal mock key store that always accepts any token.""" + store = AsyncMock() + store.lookup_by_token = AsyncMock( + return_value=ApiKeyInfo(key_id=uuid4(), scopes=scopes) + ) + return store + + +def _make_vector_store(results: list[SearchResult]) -> AsyncMock: + store = AsyncMock() + store.search = AsyncMock(return_value=results) + return store + + +def _make_embedding_provider() -> AsyncMock: + provider = AsyncMock() + provider.embed_query = AsyncMock(return_value=[0.1, 0.2, 0.3]) + return provider + + +def _result(chunk_id: str = "chunk-1", score: float = 0.9) -> SearchResult: + return SearchResult( + chunk_id=chunk_id, + score=score, + text_snippet="Art. 2 diritti inviolabili", + document_id=uuid4(), + document_version=1, + metadata={}, + ) + + +def _make_app( + vector_store: AsyncMock, + sparse_provider: AsyncMock | None = None, +) -> FastAPI: + """Build a FastAPI test app with the index router and a stub registry.""" + from vektra_index.api import router + + app = FastAPI() + app.state.registry = ProviderRegistry() + app.state.registry.register("key_store", "default", _make_key_store(["query"])) + app.state.registry.register("embedding", "default", _make_embedding_provider()) + app.state.registry.register("vector_store", "default", vector_store) + if sparse_provider is not None: + app.state.registry.register("sparse_embedding", "default", sparse_provider) + app.include_router(router) + return app + + +async def _post_search(app: FastAPI, body: dict) -> tuple[int, dict]: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.post( + "/api/v1/search", + json=body, + headers={"Authorization": "Bearer test-token"}, + ) + return resp.status_code, resp.json() + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_search_uses_registry_vector_store() -> None: + """Results must come from the registry's vector store, not pgvector.""" + vector_store = _make_vector_store([_result()]) + app = _make_app(vector_store) + + status, data = await _post_search( + app, + { + "query": "diritti", + "namespace": "default", + "top_k": 5, + "search_mode": "dense", + }, + ) + + assert status == 200 + assert data["total"] == 1 + assert data["results"][0]["chunk_id"] == "chunk-1" + + vector_store.search.assert_awaited_once() + kwargs = vector_store.search.await_args.kwargs + assert kwargs["namespace"] == "default" + assert kwargs["top_k"] == 5 + assert kwargs["search_mode"] == SearchMode.DENSE + assert kwargs["query_embedding"].dense == [0.1, 0.2, 0.3] + + +@pytest.mark.asyncio +async def test_search_hybrid_falls_back_to_dense_without_sparse() -> None: + """HYBRID with no registered sparse provider degrades to DENSE.""" + vector_store = _make_vector_store([]) + app = _make_app(vector_store, sparse_provider=None) + + status, data = await _post_search( + app, + {"query": "q", "namespace": "default", "top_k": 3, "search_mode": "hybrid"}, + ) + + assert status == 200 + assert data["results"] == [] + kwargs = vector_store.search.await_args.kwargs + assert kwargs["search_mode"] == SearchMode.DENSE + assert kwargs["query_embedding"].sparse is None + + +@pytest.mark.asyncio +async def test_search_hybrid_uses_registered_sparse_provider() -> None: + """HYBRID with a registered sparse provider keeps hybrid mode and sparse vector.""" + vector_store = _make_vector_store([_result()]) + sparse_vector = MagicMock(name="sparse_vector") + sparse_provider = AsyncMock() + sparse_provider.embed_query = AsyncMock(return_value=sparse_vector) + app = _make_app(vector_store, sparse_provider=sparse_provider) + + status, _ = await _post_search( + app, + {"query": "q", "namespace": "default", "top_k": 3, "search_mode": "hybrid"}, + ) + + assert status == 200 + sparse_provider.embed_query.assert_awaited_once_with("q") + kwargs = vector_store.search.await_args.kwargs + assert kwargs["search_mode"] == SearchMode.HYBRID + assert kwargs["query_embedding"].sparse is sparse_vector diff --git a/vektra-index/tests/test_pgvector_unit.py b/vektra-index/tests/test_pgvector_unit.py index 19233c1d..e8e8e06f 100644 --- a/vektra-index/tests/test_pgvector_unit.py +++ b/vektra-index/tests/test_pgvector_unit.py @@ -163,3 +163,144 @@ async def test_custom_index_version_used(self): provider = PgvectorProvider(active_index_version=2) assert provider._active_index_version == 2 + + +class TestPgvectorParentChunks: + """Deterministic ids, parent linkage, search exclusion, retrieve (FEAT-017).""" + + def _make_session(self): + session = AsyncMock() + session.flush = AsyncMock() + session.execute = AsyncMock() + session.add = MagicMock() + return session + + @pytest.mark.asyncio + async def test_store_honors_deterministic_ids_and_parent_linkage(self): + """store() keeps caller-provided uuid ids and fills the parent_id column.""" + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + parent_uuid = uuid4() + child_uuid = uuid4() + chunks = [ + ChunkEmbedding( + chunk_id=str(parent_uuid), + text="parent", + dense=[0.1] * 384, + metadata={"chunk_level": "parent"}, + ), + ChunkEmbedding( + chunk_id=str(child_uuid), + text="child", + dense=[0.1] * 384, + metadata={"chunk_level": "child"}, + parent_id=str(parent_uuid), + ), + ] + + with patch("vektra_index.models.DocumentChunkOrm") as MockOrm: + MockOrm.return_value = MagicMock() + result = await provider.store(session, "default", uuid4(), chunks) + + assert result == [str(parent_uuid), str(child_uuid)] + orm_kwargs = [c.kwargs for c in MockOrm.call_args_list] + assert orm_kwargs[0]["id"] == parent_uuid + assert orm_kwargs[0]["parent_id"] is None + assert orm_kwargs[1]["id"] == child_uuid + assert orm_kwargs[1]["parent_id"] == parent_uuid + + @pytest.mark.asyncio + async def test_store_falls_back_to_random_id_on_non_uuid(self): + """Non-UUID caller ids (external store-chunks callers) get random uuids.""" + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + chunks = [ + ChunkEmbedding( + chunk_id="not-a-uuid", + text="x", + dense=[0.1] * 384, + metadata={}, + parent_id="also-not-a-uuid", + ), + ] + + with patch("vektra_index.models.DocumentChunkOrm") as MockOrm: + MockOrm.return_value = MagicMock() + result = await provider.store(session, "default", uuid4(), chunks) + + UUID(result[0]) # random but valid + assert MockOrm.call_args.kwargs["parent_id"] is None + + @pytest.mark.asyncio + async def test_dense_search_excludes_parent_chunks(self): + """The dense search SQL filters out chunk_level=parent rows.""" + from sqlalchemy.dialects import postgresql + + from vektra_index.providers.pgvector import PgvectorProvider + from vektra_shared.types import QueryEmbedding + + session = self._make_session() + empty_result = MagicMock() + empty_result.all.return_value = [] + session.execute = AsyncMock(return_value=empty_result) + + provider = PgvectorProvider() + await provider.search( + session, "default", QueryEmbedding(dense=[0.1] * 384), top_k=5 + ) + + stmt = session.execute.call_args.args[0] + compiled = stmt.compile(dialect=postgresql.dialect()) + assert "IS DISTINCT FROM" in str(compiled) + # The JSONB accessor key and the excluded value are bind parameters + assert "chunk_level" in compiled.params.values() + assert "parent" in compiled.params.values() + + @pytest.mark.asyncio + async def test_retrieve_maps_rows_and_skips_invalid_ids(self): + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + parent_uuid = uuid4() + doc_id = uuid4() + row = FakeRow( + id=parent_uuid, + document_id=doc_id, + content="parent text", + chunk_metadata={"chunk_level": "parent"}, + parent_id=None, + score=0.0, + document_version=1, + ) + rows_result = MagicMock() + rows_result.all.return_value = [row] + session.execute = AsyncMock(return_value=rows_result) + + provider = PgvectorProvider() + results = await provider.retrieve( + session, "default", [str(parent_uuid), "not-a-uuid"] + ) + + assert len(results) == 1 + assert results[0].chunk_id == str(parent_uuid) + assert results[0].score == 0.0 + assert results[0].text_snippet == "parent text" + assert results[0].parent_id is None + + @pytest.mark.asyncio + async def test_retrieve_all_invalid_ids_returns_empty(self): + from vektra_index.providers.pgvector import PgvectorProvider + + session = self._make_session() + provider = PgvectorProvider() + + results = await provider.retrieve(session, "default", ["nope", "still-nope"]) + + assert results == [] + session.execute.assert_not_called() diff --git a/vektra-index/tests/test_qdrant_provider.py b/vektra-index/tests/test_qdrant_provider.py index 686f6ffd..16a60d76 100644 --- a/vektra-index/tests/test_qdrant_provider.py +++ b/vektra-index/tests/test_qdrant_provider.py @@ -244,3 +244,151 @@ async def test_unhealthy(self): status = await provider.health_check() assert status.status == "unhealthy" assert "connection refused" in status.message + + +class TestQdrantParentChunks: + """Parent chunk linkage, search exclusion, and retrieve-by-id (FEAT-017).""" + + @pytest.mark.asyncio + async def test_store_includes_parent_id_in_payload(self): + provider, _client = _make_provider() + doc_id = str(uuid4()) + + _mock_qdrant_models.PointStruct.reset_mock() + chunks = [ + ChunkEmbedding( + chunk_id="child-1", + text="child text", + dense=[0.1] * 384, + metadata={"document_id": doc_id, "chunk_level": "child"}, + parent_id="parent-1", + ), + ChunkEmbedding( + chunk_id="parent-1", + text="parent text", + dense=[0.1] * 384, + metadata={"document_id": doc_id, "chunk_level": "parent"}, + ), + ] + + await provider.store("default", chunks) + + payloads = [ + c.kwargs["payload"] for c in _mock_qdrant_models.PointStruct.call_args_list + ] + assert payloads[0]["parent_id"] == "parent-1" + assert payloads[1]["parent_id"] is None + + @pytest.mark.asyncio + async def test_build_filter_excludes_parent_chunks(self): + provider, _client = _make_provider() + + _mock_qdrant_models.Filter.reset_mock() + _mock_qdrant_models.FieldCondition.reset_mock() + provider._build_filter("default") + + filter_kwargs = _mock_qdrant_models.Filter.call_args.kwargs + assert "must_not" in filter_kwargs + assert len(filter_kwargs["must_not"]) == 1 + condition_keys = [ + c.kwargs.get("key") + for c in _mock_qdrant_models.FieldCondition.call_args_list + ] + assert "metadata.chunk_level" in condition_keys + + @pytest.mark.asyncio + async def test_search_results_carry_parent_id(self): + provider, client = _make_provider() + doc_id = str(uuid4()) + + query_result = MagicMock() + query_result.points = [ + _make_scored_point( + "child-1", + 0.9, + { + "text": "child text", + "document_id": doc_id, + "metadata": {"chunk_level": "child"}, + "namespace_id": "default", + "parent_id": "parent-1", + }, + ), + ] + client.query_points = AsyncMock(return_value=query_result) + + results = await provider.search( + "default", QueryEmbedding(dense=[0.1] * 384), 5, SearchMode.DENSE + ) + + assert results[0].parent_id == "parent-1" + + @pytest.mark.asyncio + async def test_retrieve_filters_by_namespace(self): + from types import SimpleNamespace + + provider, client = _make_provider() + doc_id = str(uuid4()) + parent_id = str(uuid4()) + alien_id = str(uuid4()) + + # Qdrant retrieve() returns Record objects with no score attribute + records = [ + SimpleNamespace( + id=parent_id, + payload={ + "text": "parent text", + "document_id": doc_id, + "metadata": {"chunk_level": "parent"}, + "namespace_id": "default", + "parent_id": None, + }, + ), + SimpleNamespace( + id=alien_id, + payload={ + "text": "other tenant", + "document_id": doc_id, + "metadata": {}, + "namespace_id": "other", + "parent_id": None, + }, + ), + ] + client.retrieve = AsyncMock(return_value=records) + + results = await provider.retrieve("default", [parent_id, alien_id]) + + client.retrieve.assert_awaited_once() + assert [r.chunk_id for r in results] == [parent_id] + assert results[0].score == 0.0 + assert results[0].text_snippet == "parent text" + + @pytest.mark.asyncio + async def test_retrieve_skips_invalid_ids(self): + """Non-UUID ids never reach the Qdrant client (it would reject the batch).""" + provider, client = _make_provider() + client.retrieve = AsyncMock(return_value=[]) + valid = str(uuid4()) + + await provider.retrieve("default", ["not-a-uuid", valid]) + + assert client.retrieve.await_args.kwargs["ids"] == [valid] + + @pytest.mark.asyncio + async def test_retrieve_empty_ids_short_circuits(self): + provider, client = _make_provider() + + results = await provider.retrieve("default", []) + + assert results == [] + client.retrieve.assert_not_called() + + @pytest.mark.asyncio + async def test_retrieve_all_invalid_ids_short_circuits(self): + provider, client = _make_provider() + + results = await provider.retrieve("default", ["parent-1", "child-2"]) + + assert results == [] + client.retrieve.assert_not_called() diff --git a/vektra-index/tests/test_tei_provider.py b/vektra-index/tests/test_tei_provider.py new file mode 100644 index 00000000..a733c48c --- /dev/null +++ b/vektra-index/tests/test_tei_provider.py @@ -0,0 +1,137 @@ +"""Unit tests for TEIEmbeddingProvider (FEAT-024). All HTTP mocked.""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +from vektra_index.providers.tei import _MAX_BATCH, TEIEmbeddingProvider + + +def _embed_handler(dim: int = 4): + """MockTransport handler: /embed returns one vector per input.""" + calls: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/embed": + inputs = json.loads(request.content)["inputs"] + calls.append(len(inputs)) + return httpx.Response(200, json=[[0.1] * dim for _ in inputs]) + return httpx.Response(404) + + return handler, calls + + +def _make_provider(handler) -> TEIEmbeddingProvider: + client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + return TEIEmbeddingProvider(url="http://tei.test", _client=client) + + +async def test_embed_query_returns_single_vector(): + handler, _ = _embed_handler(dim=4) + provider = _make_provider(handler) + vec = await provider.embed_query("hello") + assert vec == [0.1] * 4 + await provider.aclose() + + +async def test_embed_documents_chunks_to_batch_limit(): + handler, calls = _embed_handler(dim=4) + provider = _make_provider(handler) + texts = [f"doc {i}" for i in range(_MAX_BATCH + 5)] + vectors = await provider.embed_documents(texts) + assert len(vectors) == _MAX_BATCH + 5 + assert calls == [_MAX_BATCH, 5] + await provider.aclose() + + +async def test_health_check_healthy(): + handler, _ = _embed_handler() + provider = _make_provider(handler) + status = await provider.health_check() + assert status.status == "healthy" + await provider.aclose() + + +async def test_health_check_unhealthy_on_http_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + provider = _make_provider(handler) + status = await provider.health_check() + assert status.status == "unhealthy" + await provider.aclose() + + +def test_fetch_dimensions_from_info(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(200, json={"model_id": "m", "embedding_size": 1024}) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 1024 + + +def test_fetch_dimensions_probe_fallback_when_info_lacks_size(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(200, json={"model_id": "m"}) + if request.url.path == "/embed": + return httpx.Response(200, json=[[0.0] * 768]) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 768 + + +def test_fetch_dimensions_probe_fallback_when_info_denied(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response(401) + if request.url.path == "/embed": + return httpx.Response(200, json=[[0.0] * 384]) + return httpx.Response(404) + + client = httpx.Client( + transport=httpx.MockTransport(handler), base_url="http://tei.test" + ) + assert TEIEmbeddingProvider._fetch_dimensions(client) == 384 + + +async def test_auth_header_sent_when_api_key_set(): + seen: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers.get("authorization", "") + return httpx.Response(200, json=[[0.0] * 4]) + + client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + base_url="http://tei.test", + headers={"Authorization": "Bearer sekret"}, + ) + provider = TEIEmbeddingProvider( + url="http://tei.test", api_key="sekret", _client=client + ) + await provider.embed_query("q") + assert seen["auth"] == "Bearer sekret" + await provider.aclose() + + +async def test_embed_raises_on_server_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + provider = _make_provider(handler) + with pytest.raises(httpx.HTTPStatusError): + await provider.embed_documents(["doc"]) + await provider.aclose() diff --git a/vektra-ingest/pyproject.toml b/vektra-ingest/pyproject.toml index 6269f730..15e9e8ad 100644 --- a/vektra-ingest/pyproject.toml +++ b/vektra-ingest/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-ingest" -version = "0.5.1" +version = "0.6.0" description = "Document processing pipeline for the Vektra platform" readme = "README.md" requires-python = ">=3.12" @@ -21,7 +21,11 @@ dependencies = [ ] [project.optional-dependencies] -ocr = ["unstructured[pdf]>=0.15"] +# torchvision is a transitive dep (unstructured-inference -> timm) that must +# come from the pytorch-cpu index: the PyPI wheel is built against CUDA torch +# and fails at import against torch+cpu ("operator torchvision::nms does not +# exist", BUG-022). Declaring it here lets uv pin its index like torch. +ocr = ["unstructured[pdf]>=0.15", "torchvision>=0.25,<0.26"] [build-system] requires = ["hatchling"] @@ -38,6 +42,7 @@ markers = ["integration: integration tests requiring Docker (excluded from ci-un [tool.uv.sources] vektra-shared = { workspace = true } +torchvision = { index = "pytorch-cpu" } [dependency-groups] dev = [ diff --git a/vektra-ingest/src/vektra_ingest/chunking.py b/vektra-ingest/src/vektra_ingest/chunking.py index 194c67b2..2d45cd0c 100644 --- a/vektra-ingest/src/vektra_ingest/chunking.py +++ b/vektra-ingest/src/vektra_ingest/chunking.py @@ -265,6 +265,9 @@ async def _chunk_impl( parent_start = parent_end continue + # The parent carries its own hierarchy id so the ingest + # pipeline can remap these chunker-local ids to stored + # chunk ids (FEAT-017). yield DocumentChunk( text=parent_text, element_type=ElementType.TEXT, @@ -274,7 +277,7 @@ async def _chunk_impl( "token_count": len(parent_tokens), "chunk_level": "parent", }, - parent_id=None, + parent_id=parent_id, ) chunk_index += 1 diff --git a/vektra-ingest/src/vektra_ingest/pipeline.py b/vektra-ingest/src/vektra_ingest/pipeline.py index 4a3663ee..4c8be799 100644 --- a/vektra-ingest/src/vektra_ingest/pipeline.py +++ b/vektra-ingest/src/vektra_ingest/pipeline.py @@ -395,6 +395,15 @@ async def run_ingest( ), ) + # Map chunker-local hierarchy ids to stored chunk ids (FEAT-017). + # Parent chunks carry their own transient id in chunk.parent_id; + # children reference it. Stored ids are deterministic: uuid5(doc_id, position). + parent_stored_ids = { + chunk.parent_id: str(uuid5(doc_id, str(i))) + for i, chunk in enumerate(all_chunks) + if chunk.parent_id and chunk.metadata.get("chunk_level") == "parent" + } + # Build ChunkEmbedding objects with document_id in metadata _extra = extra_metadata or {} chunk_embeddings = [ @@ -411,6 +420,11 @@ async def run_ingest( "element_type": chunk.element_type.value, "position": i, }, + parent_id=( + parent_stored_ids.get(chunk.parent_id) + if chunk.parent_id and chunk.metadata.get("chunk_level") == "child" + else None + ), ) for i, (chunk, embedding, sparse) in enumerate( zip(all_chunks, embeddings, sparse_vectors) diff --git a/vektra-ingest/tests/conftest.py b/vektra-ingest/tests/conftest.py new file mode 100644 index 00000000..7f4339f7 --- /dev/null +++ b/vektra-ingest/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared fixtures for vektra-ingest unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding config defaults (e.g. VEKTRA_CHUNKING_STRATEGY=dual makes + run_ingest pick the wrong chunker under test). CI has no .env and never + sees the difference. Tests that need a specific value still set it + explicitly via constructor kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) diff --git a/vektra-ingest/tests/test_dual_chunking.py b/vektra-ingest/tests/test_dual_chunking.py index 7222dc92..d92d343a 100644 --- a/vektra-ingest/tests/test_dual_chunking.py +++ b/vektra-ingest/tests/test_dual_chunking.py @@ -74,20 +74,17 @@ async def test_text_elements_split_with_overlap(): chunks = await _collect(chunker, elements) # Should have at least one parent and at least one child - parents = [ - c for c in chunks if c.parent_id is None and c.element_type == ElementType.TEXT - ] - children = [c for c in chunks if c.parent_id is not None] + parents = [c for c in chunks if c.metadata.get("chunk_level") == "parent"] + children = [c for c in chunks if c.metadata.get("chunk_level") == "child"] assert len(parents) >= 1 assert len(children) >= 1 + # Parents carry their own hierarchy id; children reference it (FEAT-017) + parent_ids = {p.parent_id for p in parents} + assert None not in parent_ids for child in children: - assert child.parent_id is not None - assert child.metadata.get("chunk_level") == "child" - - for parent in parents: - assert parent.metadata.get("chunk_level") == "parent" + assert child.parent_id in parent_ids @pytest.mark.asyncio diff --git a/vektra-ingest/tests/test_pipeline.py b/vektra-ingest/tests/test_pipeline.py index 76236ac7..2968be15 100644 --- a/vektra-ingest/tests/test_pipeline.py +++ b/vektra-ingest/tests/test_pipeline.py @@ -711,3 +711,124 @@ async def _gen(): session=session, registry=registry, ) + + +# --------------------------------------------------------------------------- +# Parent chunk linkage at store time (FEAT-017) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_dual_ingest_links_children_to_stored_parent_ids(monkeypatch): + """Children reference the parent's deterministic stored id, not the + chunker-local transient id; parents and tables carry no parent_id.""" + from uuid import uuid5 + + from vektra_ingest.pipeline import run_ingest + + monkeypatch.setenv("VEKTRA_CHUNKING_STRATEGY", "dual") + monkeypatch.setenv("VEKTRA_PARENT_CHILD_LEVELS", "1") + + doc_id = uuid4() + + session = AsyncMock() + + async def _execute(stmt, *args, **kwargs): + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = None + return mock_result + + added_docs: list = [] + + def _session_add(obj): + added_docs.append(obj) + + async def _flush_side_effect(): + for obj in added_docs: + obj.id = doc_id + + session.execute = _execute + session.add = _session_add + session.flush = AsyncMock(side_effect=_flush_side_effect) + session.commit = AsyncMock() + + registry, mock_embedding, mock_vs = _make_registry() + mock_embedding.embed_documents = AsyncMock(return_value=[[0.1] * 384] * 4) + mock_vs.store = AsyncMock(return_value=["i0", "i1", "i2", "i3"]) + + async def _fake_extract(req): + async def _gen(): + yield DocumentChunk(text="whatever", element_type=ElementType.TEXT) + + return _gen() + + # Controlled dual output: parent + two children + standalone table + async def _fake_chunk(elements): + async def _gen(): + yield DocumentChunk( + text="parent text", + element_type=ElementType.TEXT, + metadata={"chunk_level": "parent"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="child a", + element_type=ElementType.TEXT, + metadata={"chunk_level": "child"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="child b", + element_type=ElementType.TEXT, + metadata={"chunk_level": "child"}, + parent_id="transient-1", + ) + yield DocumentChunk( + text="standalone
", + element_type=ElementType.TABLE, + metadata={}, + parent_id=None, + ) + + return _gen() + + mock_extractor = MagicMock() + mock_extractor.extract = _fake_extract + mock_extractor.supported_types.return_value = {"application/pdf"} + + with patch( + "vektra_ingest.pipeline.detect_content_type", return_value="application/pdf" + ): + with patch( + "vektra_ingest.pipeline._get_extractor", return_value=mock_extractor + ): + with patch( + "vektra_ingest.pipeline.DualStrategyChunking" + ) as mock_chunker_class: + mock_chunker = MagicMock() + mock_chunker.chunk = _fake_chunk + mock_chunker_class.return_value = mock_chunker + + result = await run_ingest( + file_content=b"%PDF-1.4 fake pdf", + filename="dual.pdf", + namespace="default", + session=session, + registry=registry, + ) + + assert result.status == "new" + stored_chunks = mock_vs.store.call_args.args[1] + assert len(stored_chunks) == 4 + + parent, child_a, child_b, table = stored_chunks + expected_parent_stored_id = str(uuid5(doc_id, "0")) + + assert parent.chunk_id == expected_parent_stored_id + assert parent.parent_id is None + assert child_a.parent_id == expected_parent_stored_id + assert child_b.parent_id == expected_parent_stored_id + assert table.parent_id is None + # Deterministic ids for every chunk: uuid5(doc_id, position) + for i, chunk in enumerate(stored_chunks): + assert chunk.chunk_id == str(uuid5(doc_id, str(i))) diff --git a/vektra-learn/pyproject.toml b/vektra-learn/pyproject.toml index 41866965..c9728987 100644 --- a/vektra-learn/pyproject.toml +++ b/vektra-learn/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-learn" -version = "0.5.1" +version = "0.6.0" description = "E-learning vertical: LMS-agnostic API and chatbot widget" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-learn/src/vektra_learn/api.py b/vektra-learn/src/vektra_learn/api.py index a5ee5ca9..16b77ef6 100644 --- a/vektra-learn/src/vektra_learn/api.py +++ b/vektra-learn/src/vektra_learn/api.py @@ -51,7 +51,11 @@ ErrorResponse, http_status_for, ) -from vektra_shared.namespace import resolve_grounding_mode, resolve_show_sources +from vektra_shared.namespace import ( + resolve_citations_enabled, + resolve_grounding_mode, + resolve_show_sources, +) from vektra_shared.types import trace_from_dict # Sentinel key_id for learn-originated conversations (JWT auth has no API key). @@ -777,6 +781,12 @@ async def course_query( else: _show_sources = _default_show_sources + # Resolve citations: namespace config > default false (FEAT-021) + if _db_factory_gm: + query_req.citations_enabled = await resolve_citations_enabled( + namespace, _db_factory_gm + ) + if registry is None: err = ErrorResponse( category=ErrorCategory.TRANSIENT, diff --git a/vektra-learn/src/vektra_learn/query.py b/vektra-learn/src/vektra_learn/query.py index 6762d63c..e9d3e92e 100644 --- a/vektra-learn/src/vektra_learn/query.py +++ b/vektra-learn/src/vektra_learn/query.py @@ -81,6 +81,7 @@ def pipeline_response_to_course_response( "score": s.score, "snippet": s.snippet, "document_name": s.document_name, # FEAT-012 + "title": s.title, # FEAT-021: citation tooltip label } for s in resp.sources ], diff --git a/vektra-learn/widget/src/chat-ui.js b/vektra-learn/widget/src/chat-ui.js index f2291075..a5f37574 100644 --- a/vektra-learn/widget/src/chat-ui.js +++ b/vektra-learn/widget/src/chat-ui.js @@ -408,6 +408,19 @@ export class ChatUI { const msgEl = msgOrStream.el || msgOrStream; if (!sources || sources.length === 0) return; + // FEAT-021: link [n] citation markers in the answer to their source. + // Marker order matches the sources order (both follow the prompt's + // budget-selected chunk order), so [n] -> sources[n-1]. + for (const cite of msgEl.querySelectorAll("sup.vektra-cite")) { + const idx = parseInt(cite.getAttribute("data-cite"), 10) - 1; + const src = sources[idx]; + const label = src && (src.title || src.document_name); + if (label) { + cite.title = label; + cite.classList.add("linked"); + } + } + const container = document.createElement("div"); container.className = "vektra-chat-sources"; diff --git a/vektra-learn/widget/src/markdown.js b/vektra-learn/widget/src/markdown.js index 960e2ca3..7375367d 100644 --- a/vektra-learn/widget/src/markdown.js +++ b/vektra-learn/widget/src/markdown.js @@ -150,5 +150,12 @@ function renderInline(text) { } return `${label} (${url})`; }) + // Citation markers [n] (FEAT-021): must run after links so [label](url) + // is already consumed. addSources() attaches the source title as a + // tooltip when the namespace has citations enabled. + .replace( + /\[(\d{1,2})\]/g, + '[$1]', + ) ); } diff --git a/vektra-learn/widget/src/styles.js b/vektra-learn/widget/src/styles.js index 944c7a3f..7696fc2d 100644 --- a/vektra-learn/widget/src/styles.js +++ b/vektra-learn/widget/src/styles.js @@ -209,6 +209,16 @@ export function buildStyles(theme) { font-weight: 600; } +.vektra-chat-msg sup.vektra-cite { + font-size: 0.75em; + vertical-align: super; + line-height: 1; +} +.vektra-chat-msg sup.vektra-cite.linked { + color: var(--vektra-primary, ${t.primary}); + cursor: help; + font-weight: 600; +} .vektra-chat-sources { margin-top: 8px; padding-top: 8px; diff --git a/vektra-shared/pyproject.toml b/vektra-shared/pyproject.toml index 1e56bd84..1e125d54 100644 --- a/vektra-shared/pyproject.toml +++ b/vektra-shared/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vektra-shared" -version = "0.5.1" +version = "0.6.0" description = "Shared protocols and types for the Vektra platform" readme = "README.md" requires-python = ">=3.12" diff --git a/vektra-shared/src/vektra_shared/config.py b/vektra-shared/src/vektra_shared/config.py index 5b602fae..894a0030 100644 --- a/vektra-shared/src/vektra_shared/config.py +++ b/vektra-shared/src/vektra_shared/config.py @@ -88,6 +88,16 @@ class EmbeddingConfig(BaseSettings): alias="VEKTRA_SPARSE_EMBEDDING_MODEL", description="Sparse embedding model name. Phase 2 only.", ) + tei_url: str = Field( + "http://localhost:8080", + alias="VEKTRA_TEI_URL", + description="TEI server base URL (native API, no /v1 suffix). Used when embedding_provider='tei' (FEAT-024).", + ) + tei_api_key: str | None = Field( + None, + alias="VEKTRA_TEI_API_KEY", + description="Bearer token for the TEI embedding server (--api-key). Optional.", + ) model_config = SettingsConfigDict( env_prefix="", extra="ignore", populate_by_name=True @@ -158,7 +168,7 @@ class RerankConfig(BaseSettings): provider: str = Field( "cross-encoder", alias="VEKTRA_RERANK_PROVIDER", - description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere'.", + description="Reranking provider: 'flashrank', 'cross-encoder', 'cohere', 'tei'.", ) model: str | None = Field( "BAAI/bge-reranker-v2-m3", @@ -171,6 +181,21 @@ class RerankConfig(BaseSettings): alias="VEKTRA_RERANK_TOP_K", description="Final top-k results after reranking.", ) + api_key: str | None = Field( + None, + alias="VEKTRA_RERANK_API_KEY", + description="API key for API-based rerank providers (e.g. 'cohere').", + ) + tei_url: str = Field( + "http://localhost:8080", + alias="VEKTRA_RERANK_TEI_URL", + description="TEI reranker server base URL (one TEI instance per model). Used when provider='tei' (FEAT-024).", + ) + tei_api_key: str | None = Field( + None, + alias="VEKTRA_RERANK_TEI_API_KEY", + description="Bearer token for the TEI reranker server. Optional.", + ) model_config = SettingsConfigDict( env_prefix="", extra="ignore", populate_by_name=True @@ -197,6 +222,24 @@ class QueryPipelineConfig(BaseSettings): alias="VEKTRA_CHUNK_DEDUP_ENABLED", description="Enable overlap deduplication for adjacent chunks from the same document.", ) + retrieval_rescue_top_k: int = Field( + 0, + ge=0, + alias="VEKTRA_RETRIEVAL_RESCUE_TOP_K", + description="When min_relevance_score empties the candidate set, keep this many top-scored chunks above the rescue floor instead of refusing (TECH-007). 0 disables rescue (default).", + ) + retrieval_rescue_floor: float = Field( + 0.02, + ge=0.0, + le=1.0, + alias="VEKTRA_RETRIEVAL_RESCUE_FLOOR", + description="Absolute minimum score for rescued chunks: candidates below this are never rescued. Only used when retrieval_rescue_top_k > 0.", + ) + parent_expansion_enabled: bool = Field( + False, + alias="VEKTRA_PARENT_EXPANSION_ENABLED", + description="Replace retrieved child chunks with their parent chunk text before prompt construction (FEAT-017). Requires dual chunking at ingest time.", + ) response_token_reserve: int = Field( 2048, ge=1, @@ -481,6 +524,8 @@ class VektraSettings(BaseSettings): sparse_embedding_model: str | None = Field( None, alias="VEKTRA_SPARSE_EMBEDDING_MODEL" ) + tei_url: str = Field("http://localhost:8080", alias="VEKTRA_TEI_URL") + tei_api_key: str | None = Field(None, alias="VEKTRA_TEI_API_KEY") # Vector store vector_store_provider: str = Field("pgvector", alias="VEKTRA_VECTOR_STORE_PROVIDER") @@ -493,6 +538,11 @@ class VektraSettings(BaseSettings): query_pipeline: str = Field("advanced", alias="VEKTRA_QUERY_PIPELINE") min_relevance_score: float = Field(0.15, alias="VEKTRA_MIN_RELEVANCE_SCORE") chunk_dedup_enabled: bool = Field(True, alias="VEKTRA_CHUNK_DEDUP_ENABLED") + retrieval_rescue_top_k: int = Field(0, alias="VEKTRA_RETRIEVAL_RESCUE_TOP_K") + retrieval_rescue_floor: float = Field(0.02, alias="VEKTRA_RETRIEVAL_RESCUE_FLOOR") + parent_expansion_enabled: bool = Field( + False, alias="VEKTRA_PARENT_EXPANSION_ENABLED" + ) response_token_reserve: int = Field(2048, alias="VEKTRA_RESPONSE_TOKEN_RESERVE") context_chunk_ratio: float = Field(0.6, alias="VEKTRA_CONTEXT_CHUNK_RATIO") prompt_templates_dir: str | None = Field(None, alias="VEKTRA_PROMPT_TEMPLATES_DIR") @@ -572,6 +622,20 @@ def validate_relevance_score(cls, v: float) -> float: raise ValueError(f"min_relevance_score must be between 0 and 1, got {v}") return v + @field_validator("retrieval_rescue_floor") + @classmethod + def validate_rescue_floor(cls, v: float) -> float: + if not 0.0 <= v <= 1.0: + raise ValueError(f"retrieval_rescue_floor must be between 0 and 1, got {v}") + return v + + @field_validator("retrieval_rescue_top_k") + @classmethod + def validate_rescue_top_k(cls, v: int) -> int: + if v < 0: + raise ValueError(f"retrieval_rescue_top_k must be >= 0, got {v}") + return v + @field_validator("prompt_grounding_mode") @classmethod def validate_prompt_grounding_mode(cls, v: str) -> str: diff --git a/vektra-shared/src/vektra_shared/namespace.py b/vektra-shared/src/vektra_shared/namespace.py index 5229be6e..8fa4a53e 100644 --- a/vektra-shared/src/vektra_shared/namespace.py +++ b/vektra-shared/src/vektra_shared/namespace.py @@ -49,6 +49,39 @@ async def resolve_grounding_mode( return default_mode +async def resolve_citations_enabled( + namespace: str, + session_factory: Any, + default_value: bool = False, +) -> bool: + """Resolve citations_enabled: namespace JSONB config > default (FEAT-021). + + There is no env var for this setting: citations are a per-namespace + deployment choice (academic/compliance contexts), default off. + + Returns *default_value* on any error (namespace not found, DB error, + non-boolean value in config). + """ + try: + async with session_factory() as session: + result = await session.execute( + text("SELECT config FROM namespaces WHERE id = :ns"), + {"ns": namespace}, + ) + row = result.scalar_one_or_none() + if row and isinstance(row, dict): + ns_value = row.get("citations_enabled") + if isinstance(ns_value, bool): + return ns_value + except Exception as exc: + log.debug( + "citations_enabled_resolution_fallback", + namespace=namespace, + error=str(exc), + ) + return default_value + + async def resolve_show_sources( namespace: str, session_factory: Any, diff --git a/vektra-shared/src/vektra_shared/protocols.py b/vektra-shared/src/vektra_shared/protocols.py index 7c633f11..1415d9bc 100644 --- a/vektra-shared/src/vektra_shared/protocols.py +++ b/vektra-shared/src/vektra_shared/protocols.py @@ -117,6 +117,12 @@ async def search( raw_filters: dict[str, Any] | None = None, ) -> list[SearchResult]: ... + async def retrieve( + self, + namespace: str, + chunk_ids: list[str], + ) -> list[SearchResult]: ... + async def delete(self, namespace: str, ids: list[str]) -> int: ... async def health_check(self) -> HealthStatus: ... diff --git a/vektra-shared/src/vektra_shared/types.py b/vektra-shared/src/vektra_shared/types.py index d1d90342..4cc711a1 100644 --- a/vektra-shared/src/vektra_shared/types.py +++ b/vektra-shared/src/vektra_shared/types.py @@ -121,6 +121,7 @@ class ChunkEmbedding: dense: list[float] sparse: SparseVector | None = None # Phase 2: hybrid search metadata: dict[str, Any] = field(default_factory=dict) + parent_id: str | None = None # stored chunk_id of the parent chunk (FEAT-017) # --------------------------------------------------------------------------- @@ -157,6 +158,7 @@ class SearchResult: document_version: int = 1 # from SourceDocument.version (REQ-056) metadata: dict[str, Any] = field(default_factory=dict) original_score: float | None = None # pre-reranker score (BUG-015) + parent_id: str | None = None # stored chunk_id of the parent chunk (FEAT-017) @dataclass @@ -271,6 +273,7 @@ class QueryRequest: filters: SearchFilters | None = None stream: bool = False grounding_mode: str = "strict" # "strict" | "hybrid" (FEAT-020) + citations_enabled: bool = False # per-namespace inline citations (FEAT-021) @dataclass @@ -288,6 +291,10 @@ class SourceRef: # documents (REQ-057) appear with an " (archived)" suffix; None only when # the DB lookup failed (transient error, DB not initialised in tests). document_name: str | None = None + # FEAT-021: human-readable source title ("filename, p.N") matching the + # title attribute rendered in the prompt context; set only when the + # namespace has citations_enabled. + title: str | None = None @dataclass diff --git a/vektra-shared/tests/conftest.py b/vektra-shared/tests/conftest.py new file mode 100644 index 00000000..20fe69aa --- /dev/null +++ b/vektra-shared/tests/conftest.py @@ -0,0 +1,25 @@ +"""Shared fixtures for vektra-shared unit tests.""" + +from __future__ import annotations + +import os + +import pytest + +# External provider keys read by ExternalApiKeys (no VEKTRA_ prefix). +_EXTERNAL_API_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY") + + +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Scrub ambient config vars so default assertions stay hermetic (DEBT-025). + + When the full suite runs on a dev machine, imports during collection + (litellm calls dotenv.load_dotenv) leak the local .env into os.environ, + overriding the defaults asserted by test_config.py. CI has no .env and + never sees the difference. Tests that need a specific value still set it + explicitly via constructor kwargs or monkeypatch.setenv. + """ + for name in list(os.environ): + if name.startswith("VEKTRA_") or name in _EXTERNAL_API_KEYS: + monkeypatch.delenv(name) diff --git a/vektra-shared/tests/test_config.py b/vektra-shared/tests/test_config.py index 9a3dd1fd..e307bdc5 100644 --- a/vektra-shared/tests/test_config.py +++ b/vektra-shared/tests/test_config.py @@ -77,6 +77,27 @@ def test_eval_mode_from_env(self) -> None: cfg = QueryPipelineConfig(VEKTRA_EVAL_MODE=True) assert cfg.eval_mode is True + def test_rescue_defaults_off(self) -> None: + cfg = QueryPipelineConfig() + assert cfg.retrieval_rescue_top_k == 0 + assert cfg.retrieval_rescue_floor == 0.02 + + def test_rescue_from_env(self) -> None: + cfg = QueryPipelineConfig( + VEKTRA_RETRIEVAL_RESCUE_TOP_K=3, + VEKTRA_RETRIEVAL_RESCUE_FLOOR=0.005, + ) + assert cfg.retrieval_rescue_top_k == 3 + assert cfg.retrieval_rescue_floor == 0.005 + + def test_rescue_top_k_negative_invalid(self) -> None: + with pytest.raises(ValidationError, match="VEKTRA_RETRIEVAL_RESCUE_TOP_K"): + QueryPipelineConfig(VEKTRA_RETRIEVAL_RESCUE_TOP_K=-1) + + def test_rescue_floor_above_one_invalid(self) -> None: + with pytest.raises(ValidationError, match="VEKTRA_RETRIEVAL_RESCUE_FLOOR"): + QueryPipelineConfig(VEKTRA_RETRIEVAL_RESCUE_FLOOR=1.5) + def test_debug_log_queries_from_env(self) -> None: cfg = QueryPipelineConfig(VEKTRA_DEBUG_LOG_QUERIES=True) assert cfg.debug_log_queries is True @@ -202,6 +223,14 @@ def test_context_chunk_ratio_one_invalid(self) -> None: with pytest.raises(ValidationError, match="context_chunk_ratio"): self._make(VEKTRA_CONTEXT_CHUNK_RATIO=1.0) + def test_rescue_top_k_negative_invalid(self) -> None: + with pytest.raises(ValidationError, match="retrieval_rescue_top_k"): + self._make(VEKTRA_RETRIEVAL_RESCUE_TOP_K=-1) + + def test_rescue_floor_above_one_invalid(self) -> None: + with pytest.raises(ValidationError, match="retrieval_rescue_floor"): + self._make(VEKTRA_RETRIEVAL_RESCUE_FLOOR=1.5) + def test_min_relevance_score_valid_zero(self) -> None: s = self._make(VEKTRA_MIN_RELEVANCE_SCORE=0.0) assert s.min_relevance_score == 0.0 diff --git a/vektra-shared/tests/test_namespace.py b/vektra-shared/tests/test_namespace.py index 7dc03c81..163f8468 100644 --- a/vektra-shared/tests/test_namespace.py +++ b/vektra-shared/tests/test_namespace.py @@ -4,7 +4,10 @@ import pytest -from vektra_shared.namespace import resolve_grounding_mode +from vektra_shared.namespace import ( + resolve_citations_enabled, + resolve_grounding_mode, +) @pytest.mark.asyncio @@ -92,3 +95,65 @@ async def test_returns_default_for_invalid_mode_in_config(): "test-ns", mock_factory, default_mode="strict" ) assert result == "strict" + + +# --- citations_enabled tests (FEAT-021) --- + + +@pytest.mark.asyncio +async def test_citations_enabled_from_namespace_config(): + """Namespace config.citations_enabled=true overrides the default.""" + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"citations_enabled": True} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is True + + +@pytest.mark.asyncio +async def test_citations_enabled_defaults_false_when_key_missing(): + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"grounding_mode": "strict"} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False + + +@pytest.mark.asyncio +async def test_citations_enabled_ignores_non_bool_value(): + """A truthy non-bool ("yes", 1) in config must not enable citations.""" + mock_session = AsyncMock() + mock_result = MagicMock() + mock_result.scalar_one_or_none.return_value = {"citations_enabled": "yes"} + mock_session.execute.return_value = mock_result + + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock(return_value=mock_session) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False + + +@pytest.mark.asyncio +async def test_citations_enabled_defaults_false_on_db_error(): + mock_factory = MagicMock() + mock_factory.return_value.__aenter__ = AsyncMock( + side_effect=RuntimeError("db down") + ) + mock_factory.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await resolve_citations_enabled("test-ns", mock_factory) + assert result is False