feat: Core AI Pipeline, Vector Relationships, and E2E Dockerization

app/main.py

@@ -1,10 +1,30 @@
 from fastapi import FastAPI
+from contextlib import asynccontextmanager
+import logging
+
 from app.routes import search
 from app.routes import upload
 from app.routes import files 
+from app.processing.cache import ModelCache  # <-- Import the cache
+
+logger = logging.getLogger(__name__)
+
+# --- THE PRE-WARMER ---
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logger.info(" Server booting up. Pre-loading AI models into RAM...")
+    ModelCache.get_encoder()
+    ModelCache.get_ocr_reader()
+    logger.info(" Models loaded successfully. Ready for instant bulk ingestion.")
+
+    yield  # The server handles actual user requests here
+
+    logger.info("Shutting down server and clearing RAM.")
+# ----------------------
 
 app = FastAPI(
-    title="WhereTF Backend"
+    title="WhereTF Backend",
+    lifespan=lifespan  # <-- Attach the hook here
 )
 
 app.include_router(search.router)
@@ -13,11 +33,8 @@
 
 @app.get("/health", tags=["System"])
 def health_check():
-    """
-    Health check endpoint to verify the API is running.
-    """
     return {
         "status": "healthy",
         "service": "WhereTF Backend",
-        "database_connected": True  # If the API booted, the Docker depends_on guarantees this is true
-    }
+        "database_connected": True 
+    }

app/models/relationship.py

@@ -1,21 +1,21 @@
 import uuid
-from sqlalchemy import Float, String, ForeignKey, UniqueConstraint
-from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.sql import func
-from sqlalchemy.orm import Mapped, mapped_column
-
+from sqlalchemy import Float, ForeignKey, UniqueConstraint
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 from app.database import Base
 
 class FileRelationship(Base):
     __tablename__ = "file_relationships"
 
     id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid())
+
     source_file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("files.id", ondelete="CASCADE"))
     target_file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("files.id", ondelete="CASCADE"))
 
     similarity_score: Mapped[float] = mapped_column(Float, nullable=False)
-    relation_type: Mapped[str] = mapped_column(String(50), server_default='semantic_similarity')
 
+    #  Preventing the duplicate relationship pairs
     __table_args__ = (
         UniqueConstraint('source_file_id', 'target_file_id', name='uq_file_relationship'),
     )

app/processing/cache.py

@@ -0,0 +1,35 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+class ModelCache:
+    _encoder = None
+    _ocr_reader = None
+
+    @classmethod
+    def get_encoder(cls):
+        if cls._encoder is None:
+            try:
+                from sentence_transformers import SentenceTransformer
+                logger.info("[System] Loading Sentence Transformer into RAM...")
+                #  loading it exactly once here
+                cls._encoder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+                logger.info("[System] Sentence Transformer loaded successfully.")
+            except ImportError as exc:
+                raise RuntimeError("pip install sentence-transformers") from exc
+        return cls._encoder
+
+    @classmethod
+    def get_ocr_reader(cls, languages=("en",)):
+        if cls._ocr_reader is None:
+            try:
+                import easyocr
+                import torch
+                # Safely check for GPU, otherwise fallback to CPU
+                gpu_available = torch.cuda.is_available()
+                logger.info(f"[System] Loading EasyOCR into RAM (gpu={gpu_available})...")
+                cls._ocr_reader = easyocr.Reader(list(languages), gpu=gpu_available)
+                logger.info("[System] EasyOCR loaded successfully.")
+            except ImportError as exc:
+                raise RuntimeError("pip install easyocr torch") from exc
+        return cls._ocr_reader

app/processing/embeddings.py

@@ -14,9 +14,10 @@
 from __future__ import annotations
 
 import logging
-from functools import lru_cache
 from typing import TYPE_CHECKING
 
+from .cache import ModelCache  # <-- Importing our new Singleton
+
 if TYPE_CHECKING:
     from .extractors import ChunkData
 
@@ -26,27 +27,6 @@
 DEFAULT_BATCH_SIZE = 64
 
 
-# ---------------------------------------------------------------------------
-# Lazy model singleton
-# ---------------------------------------------------------------------------
-
-@lru_cache(maxsize=1)
-def _get_model():
-    """Load and cache the SentenceTransformer model (once per process)."""
-    try:
-        from sentence_transformers import SentenceTransformer  # type: ignore
-    except ImportError as exc:
-        raise RuntimeError(
-            "sentence-transformers is required. "
-            "Install it with:  pip install sentence-transformers"
-        ) from exc
-
-    logger.info("[embeddings] Loading model '%s' …", MODEL_NAME)
-    model = SentenceTransformer(MODEL_NAME)
-    logger.info("[embeddings] Model loaded.")
-    return model
-
-
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -83,7 +63,8 @@ def embed_chunks(
     if not pending_indices:
         return chunks
 
-    model = _get_model()
+    # grabs the model from RAM using singleton
+    model = ModelCache.get_encoder()
     texts = [chunks[i]["content_text"] for i in pending_indices]
 
     logger.info(
@@ -106,4 +87,4 @@ def embed_chunks(
         chunks[chunk_idx]["embedding"] = vectors[list_pos].tolist()
 
     logger.info("[embeddings] Done.")
-    return chunks
+    return chunks

app/processing/expansion.py

@@ -0,0 +1,54 @@
+import json
+import urllib.request
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Ollama's url
+OLLAMA_URL = "http://ollama:11434/api/generate"
+
+MODEL_NAME = "qwen2:0.5b" 
+
+def generate_hypothetical_document(query: str) -> str:
+    """
+    Uses a local LLM via Ollama to generate a hypothetical textbook answer 
+    for the user's query. This is the core of the HyDE algorithm.
+    """
+    # The system prompt forces the LLM to write like a document in your database,
+    # rather than acting like a chat assistant.
+    prompt = (
+        "You are an expert technical writer. Write a single, concise, factual, "
+        "and highly technical textbook paragraph that directly answers the following "
+        "search query. Do not include conversational filler, introductions, or conclusions. "
+        f"Query: {query}"
+    )
+
+    payload = {
+        "model": MODEL_NAME,
+        "prompt": prompt,
+        "stream": False,
+        "options": {
+            "temperature": 0.3  # Low temperature forces deterministic, factual generation
+        }
+    }
+
+    try:
+        logger.info(f"[expansion] Requesting hypothetical document for: '{query}'")
+
+        # Zero-dependency HTTP request using standard Python
+        req = urllib.request.Request(
+            OLLAMA_URL, 
+            data=json.dumps(payload).encode('utf-8'), 
+            headers={'Content-Type': 'application/json'}
+        )
+
+        with urllib.request.urlopen(req, timeout=120) as response:
+            result = json.loads(response.read().decode('utf-8'))
+            hypothetical_doc = result.get("response", "").strip()
+
+            logger.info(f"[expansion] HyDE Generated: {hypothetical_doc[:100]}...")
+            return hypothetical_doc
+
+    except Exception as e:
+        logger.warning(f"[expansion] Local LLM failed: {e}. Falling back to original query.")
+        return query

app/processing/ocr.py

@@ -17,6 +17,7 @@
 import logging
 from functools import lru_cache
 from typing import Sequence
+from .cache import ModelCache  # added a new import 
 
 logger = logging.getLogger(__name__)
 
@@ -85,7 +86,8 @@ def ocr_image_bytes(
         import numpy as np  # type: ignore
         from PIL import Image  # type: ignore
 
-        reader = _get_reader(tuple(languages))
+        #reader = _get_reader(tuple(languages))
+        reader = ModelCache.get_ocr_reader(tuple(languages))
 
         # Convert bytes → PIL → numpy (EasyOCR accepts numpy arrays natively)
         pil_img = Image.open(io.BytesIO(image_bytes)).convert("RGB")

app/processing/search.py

@@ -34,7 +34,8 @@
 import logging
 from typing import Any, Literal
 
-from .embeddings import _get_model
+from .cache import ModelCache
+from .expansion import generate_hypothetical_document
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +85,7 @@ def embed_query(query: str) -> list[float]:
         384-dimensional unit vector, ready to be cast to pgvector's
         ``vector`` type.
     """
-    model = _get_model()
+    model = ModelCache.get_encoder()
     vec = model.encode(
         [query],
         convert_to_numpy=True,
@@ -284,27 +285,31 @@ def build_query(
     """
     logger.info("[search] Building %s query: %r (top_k=%d)", mode, query, top_k)
 
-    # 1. Embed
-    vector = embed_query(query)
+    # ---> NEW: Intercept and Expand <---
+    # Only generate a hypothetical document if we are doing vector math
+    if mode in ("vector", "hybrid"):
+        expanded_text_for_vector = generate_hypothetical_document(query)
+    else:
+        expanded_text_for_vector = query
 
-    # 2. Build SQL variants (pre-build all three so the API can expose them)
+    # 1. Embed (Using the HYPOTHETICAL document)
+    vector = embed_query(expanded_text_for_vector)
+
+    # 2. Build SQL variants
     has_filter = file_filter is not None
     sql = {
         "vector":  _sql_vector(top_k, has_filter),
         "keyword": _sql_keyword(top_k, has_filter),
         "hybrid":  _sql_hybrid(top_k, has_filter, rrf_k=rrf_k),
     }
 
-    # 3. Build bind parameters for the chosen mode
+    # 3. Build bind parameters
     params: dict[str, Any] = {"top_k": top_k}
 
     needs_vec  = mode in ("vector", "hybrid")
     needs_text = mode in ("keyword", "hybrid")
 
     if needs_vec:
-        # pgvector accepts the vector as a Python list; SQLAlchemy passes it
-        # through.  If you hit type errors, cast explicitly:
-        # str(vector)  →  "[0.12, -0.34, …]"  (pgvector's text literal)
         params["query_vec"] = str(vector)
 
     if needs_text:
@@ -315,14 +320,14 @@ def build_query(
 
     return SearchPayload(
         query=query,
+        expanded_query=expanded_text_for_vector,
         vector=vector,
         mode=mode,
         top_k=top_k,
         sql=sql,
         params=params,
     )
 
-
 # ---------------------------------------------------------------------------
 # Convenience: just get the vector (for callers that run their own SQL)
 # ---------------------------------------------------------------------------

app/routes/files.py

@@ -6,7 +6,7 @@
 from sqlalchemy import select
 from pydantic import BaseModel
 from typing import List, Optional
-
+from app.models.relationship import FileRelationship
 from app.database import get_db
 from app.models import File
 
@@ -66,4 +66,36 @@ def delete_file(file_id: uuid.UUID, db: Session = Depends(get_db)):
     db.delete(db_file)
     db.commit()
 
-    return {"status": "success", "message": f"File and vectors wiped successfully."}
+    return {"status": "success", "message": f"File and vectors wiped successfully."}
+
+# 4. File relationship
+@router.get("/{file_id}/related")
+def get_related_files(file_id: uuid.UUID, db: Session = Depends(get_db)):
+    """Returns a list of files mathematically related to the requested file."""
+
+    # Check if the file exists
+    db_file = db.scalar(select(File).where(File.id == file_id))
+    if not db_file:
+        raise HTTPException(status_code=404, detail="File not found")
+
+    # Fetch relationships where this file is the source
+    relationships = db.execute(
+        select(FileRelationship, File)
+        .join(File, FileRelationship.target_file_id == File.id)
+        .where(FileRelationship.source_file_id == file_id)
+        .order_by(FileRelationship.similarity_score.desc())
+    ).all()
+
+    return {
+        "status": "success",
+        "file_id": str(file_id),
+        "related_files": [
+            {
+                "target_file_id": str(rel.FileRelationship.target_file_id),
+                "similarity_score": round(rel.FileRelationship.similarity_score, 4),
+                "file_path": rel.File.file_path,
+                "mime_type": rel.File.mime_type
+            }
+            for rel in relationships
+        ]
+    }