diff --git a/app/processing/__init__.py b/app/processing/__init__.py index 2a5646d..f2ad246 100644 --- a/app/processing/__init__.py +++ b/app/processing/__init__.py @@ -6,22 +6,37 @@ Public surface ~~~~~~~~~~~~~~ +Indexing (ingestion side) from processing import process file_rows, content_rows = process("/path/to/scan") - # file_rows → list of dicts matching the `File` SQLAlchemy model - # content_rows → list of dicts matching the `FileContent` model + # file_rows → list of dicts matching the `File` SQLAlchemy model + # content_rows → list of dicts matching the `FileContent` SQLAlchemy model -Lower-level modules -~~~~~~~~~~~~~~~~~~~ -* ``traversal`` – recursive file walker -* ``extractors`` – per-format text + embedded-OCR extractors -* ``ocr`` – EasyOCR singleton wrapper +Searching (query side) + from processing.search import build_query, query_vector + + payload = build_query("JWT authentication", mode="hybrid", top_k=10) + # payload["sql"]["hybrid"] → parametrised SQL string + # payload["params"] → bind-param dict for session.execute() + # payload["vector"] → 384-dim list[float] + + vec = query_vector("what is time management") # just the embedding + +Modules +~~~~~~~ +* ``traversal`` – recursive file walker → FileEntry dicts +* ``extractors`` – per-format text + embedded-OCR extractors, rolling chunker +* ``ocr`` – lazy EasyOCR singleton (GPU-aware) * ``embeddings`` – batched sentence-transformer inference -* ``pipeline`` – top-level orchestrator (re-exported as ``process``) +* ``pipeline`` – top-level orchestrator (re-exported as ``process``) +* ``search`` – query embedding + SQL builder for vector/keyword/hybrid search + +See ``BACKEND_INTEGRATION.md`` for the full setup guide. """ from .pipeline import process +from .search import build_query, query_vector -__all__ = ["process"] +__all__ = ["process", "build_query", "query_vector"] \ No newline at end of file diff --git a/app/processing/search.py b/app/processing/search.py new file mode 100644 index 0000000..d1350b5 --- /dev/null +++ b/app/processing/search.py @@ -0,0 +1,339 @@ +""" +search.py +--------- +Query-side search utilities for WhereTF. + +This module is the mirror of pipeline.py: instead of indexing files it +turns a user query into everything the database layer needs to run a +search. + +Three search modes are supported, all composable: + + VECTOR – semantic similarity via pgvector cosine distance + KEYWORD – full-text search via PostgreSQL tsvector / tsquery + HYBRID – reciprocal-rank fusion of both lists (best for most UIs) + +All functions return plain Python dicts / lists — no SQLAlchemy, no DB +connections anywhere in this file. The backend engineer only needs to +paste the returned SQL + parameters into their session.execute() call. + +Quick-start +~~~~~~~~~~~ + from processing.search import build_query + + payload = build_query("machine learning time series", mode="hybrid", top_k=10) + + # payload["vector"] → list[float] (384 dims) — bind to :query_vec + # payload["query"] → str — bind to :query_text + # payload["sql"] → dict with keys vector / keyword / hybrid + # payload["params"] → dict ready for session.execute(text(sql), params) +""" + +from __future__ import annotations + +import logging +from typing import Any, Literal + +from .embeddings import _get_model + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Types +# --------------------------------------------------------------------------- + +SearchMode = Literal["vector", "keyword", "hybrid"] + +SearchPayload = dict[str, Any] +""" +{ + "query" : str – original user query string + "vector" : list[float] – 384-dim embedding of the query + "mode" : SearchMode + "top_k" : int + "sql" : { + "vector" : str, – SQL for vector-only search + "keyword" : str, – SQL for keyword-only search + "hybrid" : str, – SQL for hybrid RRF search + } + "params" : dict – bind parameters for the chosen mode's SQL +} +""" + + +# --------------------------------------------------------------------------- +# Step 1 — embed the query +# --------------------------------------------------------------------------- + +def embed_query(query: str) -> list[float]: + """ + Encode a single query string with the same model used at index time. + + The vector is unit-normalised so cosine similarity == dot product, + matching the normalisation applied during ingestion. + + Parameters + ---------- + query : str + Raw user query. Preprocessing (lowercasing, stopword removal) + is intentionally skipped — the model handles it internally. + + Returns + ------- + list[float] + 384-dimensional unit vector, ready to be cast to pgvector's + ``vector`` type. + """ + model = _get_model() + vec = model.encode( + [query], + convert_to_numpy=True, + normalize_embeddings=True, + ) + return vec[0].tolist() + + +# --------------------------------------------------------------------------- +# Step 2 — build SQL for each search mode +# --------------------------------------------------------------------------- + +# ── 2a. Vector search ─────────────────────────────────────────────────────── + +def _sql_vector(top_k: int, file_filter: bool) -> str: + """ + Pure cosine-similarity search using pgvector's <=> operator. + + The <=> operator returns cosine *distance* (0 = identical, 2 = opposite) + so ORDER BY ASC gives the most similar results first. + + An HNSW or IVFFlat index on FileContent.embedding makes this O(log N). + """ + where = "AND f.file_path LIKE :file_filter" if file_filter else "" + return f""" +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + 1 - (fc.embedding <=> CAST(:query_vec AS vector)) AS score +FROM file_content fc +JOIN file f ON f.id = fc.file_id +WHERE 1=1 {where} +ORDER BY fc.embedding <=> CAST(:query_vec AS vector) ASC +LIMIT :top_k; +""".strip() + + +# ── 2b. Keyword (full-text) search ────────────────────────────────────────── + +def _sql_keyword(top_k: int, file_filter: bool) -> str: + """ + PostgreSQL full-text search against the pre-computed tsvector column + (keyword_tokens). + + ts_rank_cd weights term density and cover density, which works well + for document search. The plainto_tsquery() function handles natural + language queries without requiring the user to know tsquery syntax. + """ + where = "AND f.file_path LIKE :file_filter" if file_filter else "" + return f""" +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + ts_rank_cd(fc.keyword_tokens, plainto_tsquery('english', :query_text)) AS score +FROM file_content fc +JOIN file f ON f.id = fc.file_id +WHERE fc.keyword_tokens @@ plainto_tsquery('english', :query_text) {where} +ORDER BY score DESC +LIMIT :top_k; +""".strip() + + +# ── 2c. Hybrid search (Reciprocal Rank Fusion) ────────────────────────────── + +def _sql_hybrid(top_k: int, file_filter: bool, rrf_k: int = 60) -> str: + """ + Hybrid search combining vector and keyword results via Reciprocal Rank + Fusion (RRF). + + RRF score = 1/(k + rank_vector) + 1/(k + rank_keyword) + + rrf_k=60 is the standard value from the original RRF paper (Cormack + et al., 2009). It prevents very high-ranked results from dominating + too strongly. + + Both sub-queries fetch top_k * 2 candidates so the fusion pool is + large enough to surface good cross-list results before the final LIMIT. + + Chunks that appear in only one list still score; they just score lower + than chunks that appear in both. + """ + pool = top_k * 2 + where = "AND f.file_path LIKE :file_filter" if file_filter else "" + return f""" +WITH vector_ranked AS ( + SELECT + fc.id AS chunk_id, + ROW_NUMBER() OVER ( + ORDER BY fc.embedding <=> CAST(:query_vec AS vector) ASC + ) AS rank + FROM file_content fc + JOIN file f ON f.id = fc.file_id + WHERE 1=1 {where} + LIMIT {pool} +), +keyword_ranked AS ( + SELECT + fc.id AS chunk_id, + ROW_NUMBER() OVER ( + ORDER BY ts_rank_cd( + fc.keyword_tokens, + plainto_tsquery('english', :query_text) + ) DESC + ) AS rank + FROM file_content fc + JOIN file f ON f.id = fc.file_id + WHERE fc.keyword_tokens @@ plainto_tsquery('english', :query_text) {where} + LIMIT {pool} +), +fused AS ( + SELECT + COALESCE(v.chunk_id, k.chunk_id) AS chunk_id, + COALESCE(1.0 / ({rrf_k} + v.rank), 0) + + COALESCE(1.0 / ({rrf_k} + k.rank), 0) AS rrf_score + FROM vector_ranked v + FULL OUTER JOIN keyword_ranked k ON k.chunk_id = v.chunk_id +) +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + fused.rrf_score AS score +FROM fused +JOIN file_content fc ON fc.id = fused.chunk_id +JOIN file f ON f.id = fc.file_id +ORDER BY fused.rrf_score DESC +LIMIT :top_k; +""".strip() + + +# --------------------------------------------------------------------------- +# Step 3 — public builder +# --------------------------------------------------------------------------- + +def build_query( + query: str, + *, + mode: SearchMode = "hybrid", + top_k: int = 10, + file_filter: str | None = None, + rrf_k: int = 60, +) -> SearchPayload: + """ + Turn a user query string into a ready-to-execute search payload. + + Parameters + ---------- + query : str + Natural-language search query from the user. + mode : "vector" | "keyword" | "hybrid" + Search strategy. ``"hybrid"`` (default) works best for most + queries; use ``"keyword"`` for exact-term or acronym lookups; + use ``"vector"`` for conceptual / semantic queries. + top_k : int + Maximum number of results to return. Default 10. + file_filter : str | None + Optional SQL LIKE pattern to restrict results to a subtree, e.g. + ``"/home/user/Documents/%"`` or ``"%.py"``. + Pass ``None`` (default) to search all files. + rrf_k : int + RRF constant. 60 is the standard value; increase to 120 to give + lower-ranked results more weight (flatter distribution). + + Returns + ------- + SearchPayload + A dict containing: + + ``query`` – original query string + ``vector`` – 384-dim list[float] embedding of the query + ``mode`` – the requested mode + ``top_k`` – the limit + ``sql`` – dict with keys ``"vector"``, ``"keyword"``, ``"hybrid"`` + containing the raw SQL strings for each mode + ``params`` – bind-parameter dict for the *chosen* mode's SQL, + ready for ``session.execute(text(sql), params)`` + + Example + ------- + :: + + payload = build_query("authentication JWT", mode="hybrid", top_k=5) + + from sqlalchemy import text + rows = db.execute( + text(payload["sql"]["hybrid"]), + payload["params"], + ).mappings().all() + """ + logger.info("[search] Building %s query: %r (top_k=%d)", mode, query, top_k) + + # 1. Embed + vector = embed_query(query) + + # 2. Build SQL variants (pre-build all three so the API can expose them) + has_filter = file_filter is not None + sql = { + "vector": _sql_vector(top_k, has_filter), + "keyword": _sql_keyword(top_k, has_filter), + "hybrid": _sql_hybrid(top_k, has_filter, rrf_k=rrf_k), + } + + # 3. Build bind parameters for the chosen mode + params: dict[str, Any] = {"top_k": top_k} + + needs_vec = mode in ("vector", "hybrid") + needs_text = mode in ("keyword", "hybrid") + + if needs_vec: + # pgvector accepts the vector as a Python list; SQLAlchemy passes it + # through. If you hit type errors, cast explicitly: + # str(vector) → "[0.12, -0.34, …]" (pgvector's text literal) + params["query_vec"] = str(vector) + + if needs_text: + params["query_text"] = query + + if has_filter: + params["file_filter"] = file_filter + + return SearchPayload( + query=query, + vector=vector, + mode=mode, + top_k=top_k, + sql=sql, + params=params, + ) + + +# --------------------------------------------------------------------------- +# Convenience: just get the vector (for callers that run their own SQL) +# --------------------------------------------------------------------------- + +def query_vector(query: str) -> list[float]: + """ + Minimal helper — returns only the 384-dim embedding of *query*. + + Use this when you handle the SQL yourself and just need the vector:: + + vec = query_vector("find python async examples") + # hand vec to your ORM / raw psycopg2 cursor + """ + return embed_query(query) \ No newline at end of file diff --git a/search_help.md b/search_help.md new file mode 100644 index 0000000..0b6f164 --- /dev/null +++ b/search_help.md @@ -0,0 +1,599 @@ +# WhereTF — Backend Integration Guide + +This document is for the backend engineer wiring the `processing` package +into the FastAPI + SQLAlchemy + PostgreSQL stack. + +--- + +## Table of contents + +1. [PostgreSQL setup](#1-postgresql-setup) +2. [SQLAlchemy models](#2-sqlalchemy-models) +3. [Indexing files (ingestion)](#3-indexing-files-ingestion) +4. [Searching (query side)](#4-searching-query-side) +5. [FastAPI route examples](#5-fastapi-route-examples) +6. [Performance & index tuning](#6-performance--index-tuning) +7. [End-to-end request flow](#7-end-to-end-request-flow) +8. [Troubleshooting](#8-troubleshooting) + +--- + +## 1. PostgreSQL setup + +### 1.1 Enable pgvector + +```sql +-- Run once per database, as a superuser. +CREATE EXTENSION IF NOT EXISTS vector; +``` + +### 1.2 Create the tables + +```sql +CREATE TABLE IF NOT EXISTS file ( + id SERIAL PRIMARY KEY, + file_path TEXT NOT NULL UNIQUE, + file_hash VARCHAR(64) NOT NULL, + mime_type VARCHAR(128), + last_modified TIMESTAMP, + tags TEXT[] DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS file_content ( + id SERIAL PRIMARY KEY, + file_id INTEGER NOT NULL REFERENCES file(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + content_text TEXT NOT NULL, + + -- 384-dim vector from sentence-transformers/all-MiniLM-L6-v2 + embedding vector(384), + + -- Full-text search column: auto-generated from content_text. + -- PostgreSQL fills this for you — never insert it from Python. + keyword_tokens TSVECTOR + GENERATED ALWAYS AS (to_tsvector('english', content_text)) STORED, + + UNIQUE (file_id, chunk_index) +); +``` + +### 1.3 Create indexes + +```sql +-- ── Full-text search index (GIN) ───────────────────────────────────────── +CREATE INDEX IF NOT EXISTS idx_fc_keyword + ON file_content USING GIN (keyword_tokens); + +-- ── Vector similarity index (HNSW — fastest for cosine, pgvector ≥ 0.5) ─ +-- m=16 ef_construction=64 are sensible defaults for up to ~1 M vectors. +-- Increase ef_construction (e.g. 128) for higher recall at indexing cost. +CREATE INDEX IF NOT EXISTS idx_fc_embedding_hnsw + ON file_content + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); + +-- ── Alternatively, IVFFlat (older, less RAM) ───────────────────────────── +-- Use after you have at least a few thousand rows. +-- lists ≈ sqrt(row_count) is a common heuristic. +-- CREATE INDEX idx_fc_embedding_ivf +-- ON file_content +-- USING ivfflat (embedding vector_cosine_ops) +-- WITH (lists = 100); +``` + +--- + +## 2. SQLAlchemy models + +```python +# app/models.py +from __future__ import annotations + +from datetime import datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import ( + ARRAY, Column, DateTime, ForeignKey, + Integer, String, Text, UniqueConstraint, +) +from sqlalchemy.orm import DeclarativeBase, relationship + + +class Base(DeclarativeBase): + pass + + +class File(Base): + __tablename__ = "file" + + id = Column(Integer, primary_key=True, autoincrement=True) + file_path = Column(Text, nullable=False, unique=True) + file_hash = Column(String(64), nullable=False) + mime_type = Column(String(128)) + last_modified = Column(DateTime) + tags = Column(ARRAY(Text), default=list) + + chunks: list["FileContent"] = relationship( + "FileContent", back_populates="file", cascade="all, delete-orphan" + ) + + +class FileContent(Base): + __tablename__ = "file_content" + __table_args__ = (UniqueConstraint("file_id", "chunk_index"),) + + id = Column(Integer, primary_key=True, autoincrement=True) + file_id = Column(Integer, ForeignKey("file.id", ondelete="CASCADE"), nullable=False) + chunk_index = Column(Integer, nullable=False) + content_text = Column(Text, nullable=False) + embedding = Column(Vector(384)) + # keyword_tokens is a GENERATED ALWAYS column — omit from INSERT. + + file: "File" = relationship("File", back_populates="chunks") +``` + +> **Install pgvector's SQLAlchemy adapter:** +> ```bash +> pip install pgvector +> ``` + +--- + +## 3. Indexing files (ingestion) + +```python +# app/services/indexer.py +from sqlalchemy.orm import Session +from sqlalchemy import select + +from processing import process # top-level pipeline +from app.models import File, FileContent + + +def index_path(root: str, db: Session, *, incremental: bool = True) -> dict: + """ + Walk *root*, extract + embed all content, and persist to the DB. + + Set incremental=True (default) to skip files whose SHA-256 hash + already exists in the database (unchanged files). + """ + skip_hashes: set[str] = set() + if incremental: + skip_hashes = { + row.file_hash + for row in db.execute(select(File.file_hash)).scalars() + } + + file_rows, content_rows = process(root, skip_hashes=skip_hashes) + + # ── Upsert File rows ────────────────────────────────────────────────── + for fr in file_rows: + existing = db.scalar(select(File).where(File.file_path == fr["file_path"])) + if existing: + # Re-index: drop old chunks, update metadata + existing.file_hash = fr["file_hash"] + existing.mime_type = fr["mime_type"] + existing.last_modified = fr["last_modified"] + existing.tags = fr["tags"] + else: + db.add(File(**fr)) + + db.flush() # assign IDs before inserting FK rows + + # ── Insert FileContent rows ─────────────────────────────────────────── + for cr in content_rows: + file_path = cr["file_path"] + file_obj = db.scalar(select(File).where(File.file_path == file_path)) + if file_obj is None: + continue # shouldn't happen, but guard anyway + + db.add(FileContent( + file_id = file_obj.id, + chunk_index = cr["chunk_index"], + content_text = cr["content_text"], + embedding = cr["embedding"], + # keyword_tokens: DO NOT pass — PostgreSQL generates it + )) + + db.commit() + return {"files": len(file_rows), "chunks": len(content_rows)} +``` + +--- + +## 4. Searching (query side) + +### 4.1 Build the search payload + +```python +from processing.search import build_query + +payload = build_query( + "async file processing python", + mode="hybrid", # "vector" | "keyword" | "hybrid" + top_k=10, + file_filter=None, # e.g. "/home/user/Documents/%" to restrict scope +) + +# payload["vector"] → list[float] (384 dims) +# payload["sql"]["hybrid"]→ raw SQL string +# payload["params"] → {"query_vec": "...", "query_text": "...", "top_k": 10} +``` + +### 4.2 Execute the search + +```python +from sqlalchemy import text +from sqlalchemy.orm import Session + + +def run_search(payload: dict, db: Session) -> list[dict]: + mode = payload["mode"] + sql = payload["sql"][mode] + + rows = db.execute( + text(sql), + payload["params"], + ).mappings().all() + + return [dict(row) for row in rows] +``` + +### 4.3 Raw SQL reference (copy-paste ready) + +All three queries join `file_content` → `file` and return the same five columns: +`file_path`, `mime_type`, `tags`, `chunk_index`, `content_text`, `score`. + +#### Vector search (semantic similarity) + +```sql +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + 1 - (fc.embedding <=> CAST(:query_vec AS vector)) AS score +FROM file_content fc +JOIN file f ON f.id = fc.file_id +ORDER BY fc.embedding <=> CAST(:query_vec AS vector) ASC +LIMIT :top_k; +``` + +**Bind parameters:** `:query_vec` (string form of 384-dim vector, e.g. `"[0.12, -0.34, …]"`), `:top_k` (int). + +The `<=>` operator is **cosine distance** (0 = identical). +`1 - distance` converts it to cosine **similarity** (1 = identical) for the `score` column. + +#### Keyword search (full-text) + +```sql +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + ts_rank_cd(fc.keyword_tokens, plainto_tsquery('english', :query_text)) AS score +FROM file_content fc +JOIN file f ON f.id = fc.file_id +WHERE fc.keyword_tokens @@ plainto_tsquery('english', :query_text) +ORDER BY score DESC +LIMIT :top_k; +``` + +**Bind parameters:** `:query_text` (raw query string), `:top_k`. + +`plainto_tsquery` handles natural-language input — no special syntax needed from users. +`@@` is the tsvector-match operator; it filters non-matching rows before ranking. + +#### Hybrid search — Reciprocal Rank Fusion (recommended) + +```sql +WITH vector_ranked AS ( + SELECT + fc.id AS chunk_id, + ROW_NUMBER() OVER ( + ORDER BY fc.embedding <=> CAST(:query_vec AS vector) ASC + ) AS rank + FROM file_content fc + JOIN file f ON f.id = fc.file_id + LIMIT 20 -- fetch 2× top_k candidates per arm +), +keyword_ranked AS ( + SELECT + fc.id AS chunk_id, + ROW_NUMBER() OVER ( + ORDER BY ts_rank_cd( + fc.keyword_tokens, + plainto_tsquery('english', :query_text) + ) DESC + ) AS rank + FROM file_content fc + JOIN file f ON f.id = fc.file_id + WHERE fc.keyword_tokens @@ plainto_tsquery('english', :query_text) + LIMIT 20 +), +fused AS ( + SELECT + COALESCE(v.chunk_id, k.chunk_id) AS chunk_id, + COALESCE(1.0 / (60 + v.rank), 0) + + COALESCE(1.0 / (60 + k.rank), 0) AS rrf_score + FROM vector_ranked v + FULL OUTER JOIN keyword_ranked k ON k.chunk_id = v.chunk_id +) +SELECT + f.file_path, + f.mime_type, + f.tags, + fc.chunk_index, + fc.content_text, + fused.rrf_score AS score +FROM fused +JOIN file_content fc ON fc.id = fused.chunk_id +JOIN file f ON f.id = fc.file_id +ORDER BY fused.rrf_score DESC +LIMIT :top_k; +``` + +**Bind parameters:** `:query_vec`, `:query_text`, `:top_k`. + +**Why RRF works:** A result appearing at rank 3 in vector search AND rank 5 in keyword +search gets score `1/(60+3) + 1/(60+5) ≈ 0.031`, outranking a result that only appears +at rank 1 in one list (score `1/(60+1) ≈ 0.016`). Consensus between both retrieval +methods surfaces the most reliably relevant chunks. + +--- + +## 5. FastAPI route examples + +```python +# app/routers/search.py +from __future__ import annotations + +from fastapi import APIRouter, BackgroundTasks, Depends, Query +from sqlalchemy import text +from sqlalchemy.orm import Session +from typing import Annotated, Literal + +from app.database import get_db +from app.services.indexer import index_path +from processing.search import build_query + +router = APIRouter(prefix="/api", tags=["search"]) + + +# ── GET /api/search ────────────────────────────────────────────────────────── + +@router.get("/search") +def search( + q: Annotated[str, Query(description="Natural-language query")], + mode: Annotated[Literal["vector","keyword","hybrid"], Query()] = "hybrid", + top_k: Annotated[int, Query(ge=1, le=100)] = 10, + file_filter: Annotated[str | None, Query( + description="SQL LIKE pattern, e.g. /home/user/docs/%" + )] = None, + db: Session = Depends(get_db), +): + """ + Search indexed files. Returns the top *top_k* matching chunks. + + - **vector** — semantic / conceptual queries ("how to handle auth errors") + - **keyword** — exact-term queries ("JWT", "ECONNREFUSED", acronyms) + - **hybrid** — best of both (default) + """ + payload = build_query(q, mode=mode, top_k=top_k, file_filter=file_filter) + sql = payload["sql"][mode] + rows = db.execute(text(sql), payload["params"]).mappings().all() + + return { + "query": q, + "mode": mode, + "results": [dict(r) for r in rows], + } + + +# ── POST /api/index ────────────────────────────────────────────────────────── + +@router.post("/index") +def trigger_index( + path: str, + background_tasks: BackgroundTasks, + db: Session = Depends(get_db), +): + """ + Kick off background indexing of *path* (file or directory). + Returns immediately; indexing runs asynchronously. + """ + background_tasks.add_task(index_path, path, db) + return {"status": "indexing started", "path": path} + + +# ── GET /api/search/vector-only ────────────────────────────────────────────── + +@router.get("/search/vector-only") +def search_vector( + q: Annotated[str, Query()], + top_k: Annotated[int, Query(ge=1, le=100)] = 10, + db: Session = Depends(get_db), +): + """Pure cosine-similarity search. Good for 'what is X about' queries.""" + payload = build_query(q, mode="vector", top_k=top_k) + rows = db.execute( + text(payload["sql"]["vector"]), payload["params"] + ).mappings().all() + return {"query": q, "results": [dict(r) for r in rows]} + + +# ── GET /api/search/keyword-only ───────────────────────────────────────────── + +@router.get("/search/keyword-only") +def search_keyword( + q: Annotated[str, Query()], + top_k: Annotated[int, Query(ge=1, le=100)] = 10, + db: Session = Depends(get_db), +): + """Full-text search only. Best for exact terms, error codes, acronyms.""" + payload = build_query(q, mode="keyword", top_k=top_k) + rows = db.execute( + text(payload["sql"]["keyword"]), payload["params"] + ).mappings().all() + return {"query": q, "results": [dict(r) for r in rows]} +``` + +### Response shape + +Every search endpoint returns: + +```json +{ + "query": "JWT authentication", + "mode": "hybrid", + "results": [ + { + "file_path": "/home/user/docs/auth_guide.pdf", + "mime_type": "application/pdf", + "tags": [], + "chunk_index": 3, + "content_text": "…the JWT is signed with RS256 and validated on every request…", + "score": 0.0312 + } + ] +} +``` + +--- + +## 6. Performance & index tuning + +### HNSW search-time parameter + +```python +# Set per-session before running vector queries for higher recall +# (at the cost of slightly slower queries). +db.execute(text("SET hnsw.ef_search = 100")) +``` + +Default is 40. For a local desktop app, 100–200 is fine. + +### IVFFlat: set probes at query time + +```python +# Only needed if you used IVFFlat instead of HNSW. +db.execute(text("SET ivfflat.probes = 10")) +``` + +### pgvector operators quick reference + +| Operator | Distance metric | Index ops class | +|---|---|---| +| `<=>` | Cosine distance | `vector_cosine_ops` | +| `<->` | L2 (Euclidean) | `vector_l2_ops` | +| `<#>` | Negative inner product | `vector_ip_ops` | + +Use `<=>` with unit-normalised embeddings (which WhereTF produces). + +### Useful diagnostic queries + +```sql +-- How many files and chunks are indexed? +SELECT + (SELECT COUNT(*) FROM file) AS total_files, + (SELECT COUNT(*) FROM file_content) AS total_chunks; + +-- Which file types are most common? +SELECT mime_type, COUNT(*) AS n +FROM file +GROUP BY mime_type +ORDER BY n DESC; + +-- Largest files by chunk count +SELECT f.file_path, COUNT(fc.id) AS chunks +FROM file_content fc +JOIN file f ON f.id = fc.file_id +GROUP BY f.file_path +ORDER BY chunks DESC +LIMIT 20; + +-- Verify the HNSW index exists +SELECT indexname, indexdef +FROM pg_indexes +WHERE tablename = 'file_content'; + +-- Inspect a vector search plan (should show "Index Scan using idx_fc_embedding_hnsw") +EXPLAIN (ANALYZE, BUFFERS) +SELECT id FROM file_content +ORDER BY embedding <=> '[0.1, 0.2, ...]'::vector +LIMIT 10; +``` + +--- + +## 7. End-to-end request flow + +``` +User types query in UI + │ + ▼ +GET /api/search?q=...&mode=hybrid&top_k=10 + │ + ▼ +build_query(q, mode="hybrid", top_k=10) ← processing.search + ├── embed_query(q) ← sentence-transformers model + │ └── returns list[float] (384 dims) + ├── _sql_hybrid(top_k, ...) ← builds parametrised SQL + └── returns SearchPayload {vector, sql, params} + │ + ▼ +db.execute(text(sql["hybrid"]), params) ← SQLAlchemy Session + ├── vector_ranked CTE → HNSW index scan + ├── keyword_ranked CTE → GIN index scan + ├── fused CTE → FULL OUTER JOIN + RRF score + └── final SELECT → ORDER BY rrf_score DESC, LIMIT 10 + │ + ▼ +Return JSON list of {file_path, chunk_index, content_text, score} +``` + +--- + +## 8. Troubleshooting + +### `operator does not exist: vector <=> unknown` + +The query vector is being passed as a plain Python list, not cast to `vector`. +Make sure the SQL contains `CAST(:query_vec AS vector)` — WhereTF's generated SQL +always does this, so the error usually means you wrote custom SQL without the cast. + +### `column "keyword_tokens" does not exist` + +The `GENERATED ALWAYS … STORED` syntax requires **PostgreSQL 12+**. +Run `SELECT version();` to confirm. If you're on an older version, replace the +generated column with a trigger or compute the tsvector in the application. + +### Hybrid search returns zero results + +The keyword arm of RRF requires at least one full-text match. If your query contains +only stopwords or very rare terms, `keyword_ranked` will be empty. The vector arm will +still return results; the hybrid score will just reflect vector rank only. This is +correct behaviour — RRF handles one empty arm gracefully via `COALESCE(…, 0)`. + +### Slow vector queries (no HNSW index used) + +Run `EXPLAIN ANALYZE` on your vector query. If you see `Seq Scan` instead of +`Index Scan using idx_fc_embedding_hnsw`, the planner decided a sequential scan is +cheaper (common when the table is small — under ~1,000 rows). This is fine and +actually faster at small scale. The HNSW index becomes beneficial above ~10,000 rows. + +### `pgvector` not found + +```bash +# Ubuntu / Debian +sudo apt install postgresql-16-pgvector # adjust version number + +# macOS (Homebrew) +brew install pgvector + +# Then in psql: +CREATE EXTENSION vector; +``` \ No newline at end of file