Added defensible search logic

backend/src/handlers/fileHandler.ts

@@ -194,6 +194,24 @@ const triggerExtraction = async (req: AuthenticatedRequest, res: Response): Prom
             return;
         }
 
+        // Hard guard: a private file must NEVER be sent to the worker for
+        // LLM extraction. The worker should also check isPrivate and write a
+        // stub, but a stale worker deploy could ignore the flag — so refuse
+        // here as well. Re-extraction of a private file is a no-op; the
+        // filename stub is already indexed.
+        if (fileRecord.isPrivate) {
+            logger.info(`triggerExtraction skipped for private fileId=${fileId} — already indexed as stub`);
+            fileRecord.extractionStatus = 'done';
+            fileRecord.extractionError = undefined;
+            fileRecord.chatReady = false;
+            await fileRecord.save();
+            res.status(200).json({
+                message: 'File is marked private — re-extraction would send contents to AI and is blocked. Toggle privacy off first.',
+                extraction_status: 'done',
+            });
+            return;
+        }
+
         // Reset state so the UI immediately reflects "queued" — even before
         // the worker picks the message up. Also flush the cached chat index:
         // a re-extracted file should be chunked from the new text, not the old.

backend/src/services/searchPipeline.ts

@@ -152,6 +152,16 @@ const recencyBoost = (createdAt: unknown, baseScore: number): number => {
 
 type WClient = Awaited<ReturnType<typeof getWeaviateClient>>;
 
+/** Pick alpha based on query length. Short queries (1-3 tokens) are usually
+ *  keyword-driven — bias toward BM25. Longer queries are natural language —
+ *  let dense pull more weight. */
+const pickAlpha = (query: string): number => {
+    const tokens = query.trim().split(/\s+/).filter(Boolean);
+    if (tokens.length <= 3) return 0.25; // 75% BM25, 25% dense
+    if (tokens.length <= 6) return 0.5;  // balanced
+    return 0.7; // 70% dense, 30% BM25
+};
+
 const chunkHybridSignal = async (
     client: NonNullable<WClient>,
     chunksName: string,
@@ -163,7 +173,7 @@ const chunkHybridSignal = async (
     const col = client.collections.get(chunksName);
     const res = await col.query.hybrid(query, {
         vector,
-        alpha: 0.5,
+        alpha: pickAlpha(query),
         limit: 50,
         filters: col.filter.byProperty("user_id").equal(userId),
         returnMetadata: ["score"],
@@ -180,6 +190,40 @@ const chunkHybridSignal = async (
     return [...seen.values()];
 };
 
+/** Pure BM25 over chunk text. Critical for rare-term queries like proper nouns
+ *  ("Guwahati", "Acme Corp") — hybrid's dense half dilutes exact matches,
+ *  but pure BM25 ranks the unique-term file at #1 reliably. RRF fuses this
+ *  with the hybrid signal so common-word queries still benefit from semantic. */
+const chunkBM25Signal = async (
+    client: NonNullable<WClient>,
+    chunksName: string,
+    userId: string,
+    query: string,
+): Promise<SignalHit[]> => {
+    if (!(await client.collections.exists(chunksName))) return [];
+    const col = client.collections.get(chunksName);
+    try {
+        const res = await col.query.bm25(query, {
+            queryProperties: ["chunk_text"],
+            limit: 50,
+            filters: col.filter.byProperty("user_id").equal(userId),
+            returnMetadata: ["score"],
+        });
+        const seen = new Map<string, SignalHit>();
+        let rank = 1;
+        for (const obj of res.objects) {
+            const p = obj.properties as { file_id?: string; chunk_text?: string };
+            const fid = String(p.file_id ?? "");
+            if (!fid || seen.has(fid)) { rank++; continue; }
+            seen.set(fid, { file_id: fid, rank: rank++, matched_chunk: p.chunk_text });
+        }
+        return [...seen.values()];
+    } catch (e) {
+        logger.warn(`chunkBM25Signal on ${chunksName} failed: ${e}`);
+        return [];
+    }
+};
+
 const filenameBM25Signal = async (
     client: NonNullable<WClient>,
     summaryName: string,
@@ -294,12 +338,42 @@ const entitySignal = async (
 
 // ---------- 6. Orchestrator ----------
 
-const intentWeights = (intent: Intent): { chunk: number; filename: number; summary: number; entity: number } => {
+const intentWeights = (intent: Intent): { chunk: number; chunkBM25: number; filename: number; summary: number; entity: number } => {
     // Defaults are balanced. Filename intent boosts filename + summary (where the name lives).
     // Pure content intent boosts chunk/body.
-    if (intent.filename) return { chunk: 1.0, filename: 2.5, summary: 1.5, entity: 1.5 };
-    if (intent.exactPhrases.length > 0) return { chunk: 2.0, filename: 1.0, summary: 1.5, entity: 2.0 };
-    return { chunk: 1.5, filename: 1.0, summary: 1.0, entity: 1.5 };
+    // chunkBM25 is the pure-lexical companion to chunk hybrid — weight it strongly
+    // because exact-term matches (proper nouns, IDs, rare words) are what users
+    // most often expect to win.
+    if (intent.filename) return { chunk: 1.0, chunkBM25: 1.5, filename: 2.5, summary: 1.5, entity: 1.5 };
+    if (intent.exactPhrases.length > 0) return { chunk: 1.5, chunkBM25: 2.5, filename: 1.0, summary: 1.5, entity: 2.0 };
+    return { chunk: 1.5, chunkBM25: 2.0, filename: 1.0, summary: 1.0, entity: 1.5 };
+};
+
+/** Exact-substring match post-RRF bonus. If the query token literally appears
+ *  in the file's filename, summary, or matched chunk, give a meaningful score
+ *  bump. Catches the "one file uniquely contains this term — it should be #1"
+ *  failure mode that RRF alone misses.
+ *
+ *  Scaled to ~0.05 per match, which dominates the typical RRF score
+ *  (range 0.01-0.10) without overwhelming intentional ranking. */
+const exactMatchBoost = (
+    query: string,
+    properties: Record<string, unknown>,
+    matchedChunk?: string,
+): number => {
+    const tokens = query.toLowerCase().split(/\s+/).filter((t) => t.length >= 3);
+    if (tokens.length === 0) return 0;
+    const filename = String(properties.filename ?? "").toLowerCase();
+    const summary = String(properties.summary ?? "").toLowerCase();
+    const chunk = (matchedChunk ?? "").toLowerCase();
+    let bonus = 0;
+    for (const tok of tokens) {
+        // Filename match is the strongest signal — user named the file with this word.
+        if (filename.includes(tok)) bonus += 0.08;
+        if (summary.includes(tok)) bonus += 0.04;
+        if (chunk.includes(tok)) bonus += 0.03;
+    }
+    return bonus;
 };
 
 export const runSearchPipeline = async (
@@ -337,6 +411,8 @@ export const runSearchPipeline = async (
     await Promise.all(pairs.flatMap(({ summary: summaryName, chunks: chunksName }) => [
         chunkHybridSignal(client, chunksName, userId, queryForRetrieval, queryVector)
             .then((hits) => allSignals.push({ name: "content", hits, weight: weights.chunk })),
+        chunkBM25Signal(client, chunksName, userId, queryForRetrieval)
+            .then((hits) => allSignals.push({ name: "content-bm25", hits, weight: weights.chunkBM25 })),
         filenameBM25Signal(client, summaryName, userId, queryForRetrieval)
             .then((hits) => allSignals.push({ name: "filename", hits, weight: weights.filename })),
         summaryHybridSignal(client, summaryName, userId, queryForRetrieval, queryVector)
@@ -379,17 +455,21 @@ export const runSearchPipeline = async (
         }
     }));
 
-    // Final scoring: fused RRF score + recency boost (if intent suggests it).
+    // Final scoring: fused RRF + recency boost + exact-match boost.
+    // The exact-match boost is the critical safety net for rare-term queries
+    // ("Guwahati", "INV-12345"). Without it, RRF can demote the one file that
+    // uniquely contains the term in favor of files that rank middling-but-everywhere.
     const recencyMultiplier = analysis.intent.recency ? 3 : 1;
     const results: SearchResult[] = fileIds.map((fid) => {
         const f = fused.get(fid)!;
         const parent = summariesByFile.get(fid) ?? {};
-        const boost = recencyMultiplier * recencyBoost(parent.created_at, f.score);
+        const recBoost = recencyMultiplier * recencyBoost(parent.created_at, f.score);
+        const exactBoost = exactMatchBoost(queryForRetrieval, parent, f.bestChunk);
         const entityMatch = entityMatchByFile.get(fid);
         return {
             ...parent,
             file_id: fid,
-            score: f.score + boost,
+            score: f.score + recBoost + exactBoost,
             matched_in: [...f.matched_in],
             matched_chunk: f.bestChunk,
             matched_entities: entityMatch?.entities.length ? entityMatch.entities : undefined,

smartdrive-extractor/utils/ingestion_router.py

@@ -6,6 +6,7 @@
 from smartdrive_core.mongo_status import update_status
 from .image_pipeline import process_image
 from .document_extractor import process_document
+from .weaviate_utils import save_doc_private, save_image_private
 
 logger = logging.getLogger(__name__)
 
@@ -38,6 +39,17 @@ def route_and_process(file_path: str, data: dict):
 
         update_status(str(file_id), "processing")
 
+        # Hard guard: a private file must never reach the LLM/OCR pipeline.
+        # Even if the per-processor check were missed, the router catches it.
+        if data.get("isPrivate"):
+            logger.info(f"Router: private file '{filename}' — writing filename stub only, skipping LLM/OCR")
+            if mime.startswith("image/"):
+                save_image_private(data)
+            else:
+                save_doc_private(data)
+            update_status(str(file_id), "done")
+            return {"message": f"Saved {filename} as private (content not indexed)", "created": True}
+
         if mime.startswith("image/"):
             logger.info("checking if image exists")
             if check_file_exists(IMG_COLLECTION, str(file_id), str(user_id)):

smartdrive-frontend/src/components/FileListWithDrawer.tsx

@@ -752,6 +752,8 @@ export function FileListWithDrawer({
     const [drawerOpen, setDrawerOpen] = useState(false);
 
     const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false);
+    const [privacyConfirm, setPrivacyConfirm] = useState<{ file: UploadItem; nextPrivate: boolean } | null>(null);
+    const [privacyBusy, setPrivacyBusy] = useState(false);
     const [actionById, setActionById] = useState<Record<string, Action>>({});
 
     const [previewUrl, setPreviewUrl] = useState<string | null>(null);
@@ -907,21 +909,25 @@ export function FileListWithDrawer({
         onRefresh();
     }, [bulkSelected, onRefresh]);
 
-    const handleTogglePrivacy = useCallback(async (file: UploadItem) => {
-        const nextPrivate = !file.is_private;
-        const verb = nextPrivate ? "Mark as private" : "Enable AI features";
-        const detail = nextPrivate
-            ? "This will remove the AI summary and disable chat for this file. The file itself stays uploaded."
-            : "This will re-extract the file and send its contents to the AI for summarization and indexing.";
-        if (!window.confirm(`${verb}?\n\n${detail}\n\nContinue?`)) return;
+    const requestTogglePrivacy = useCallback((file: UploadItem) => {
+        setPrivacyConfirm({ file, nextPrivate: !file.is_private });
+    }, []);
+
+    const confirmTogglePrivacy = useCallback(async () => {
+        if (!privacyConfirm) return;
+        const { file, nextPrivate } = privacyConfirm;
+        setPrivacyBusy(true);
         try {
             await apiClient.patch(`/file/${file.file_id}/privacy`, { isPrivate: nextPrivate });
             toast.success(nextPrivate ? "Marked as private." : "AI features enabled. Re-extracting…");
+            setPrivacyConfirm(null);
             onRefresh();
         } catch (err) {
             toast.error(`Could not update privacy: ${(err as Error).message || "unknown error"}`);
+        } finally {
+            setPrivacyBusy(false);
         }
-    }, [onRefresh]);
+    }, [privacyConfirm, onRefresh]);
 
     const handleRetryAllFailed = useCallback(async () => {
         const failedIds = files
@@ -1192,7 +1198,7 @@ export function FileListWithDrawer({
                             onDownload={() => handleDownload(file)}
                             onExtract={() => handleExtract(file)}
                             onToggleSelect={() => toggleSelect(file.file_id)}
-                            onTogglePrivacy={() => handleTogglePrivacy(file)}
+                            onTogglePrivacy={() => requestTogglePrivacy(file)}
                             onDeleteRequest={() => {
                                 setSelectedId(file.file_id);
                                 setConfirmDeleteOpen(true);
@@ -1456,6 +1462,67 @@ export function FileListWithDrawer({
                     </AlertDialogFooter>
                 </AlertDialogContent>
             </AlertDialog>
+
+            <AlertDialog
+                open={privacyConfirm !== null}
+                onOpenChange={(open) => { if (!open && !privacyBusy) setPrivacyConfirm(null); }}
+            >
+                <AlertDialogContent>
+                    <AlertDialogHeader>
+                        <AlertDialogTitle className="flex items-center gap-2">
+                            {privacyConfirm?.nextPrivate ? (
+                                <><IconLock size={18} /> Mark as private?</>
+                            ) : (
+                                <><IconLockOpen size={18} /> Enable AI features?</>
+                            )}
+                        </AlertDialogTitle>
+                        <AlertDialogDescription asChild>
+                            <div className="space-y-3 text-sm">
+                                <p>
+                                    <span className="font-medium text-foreground">
+                                        {privacyConfirm?.file.filename ?? "This file"}
+                                    </span>
+                                </p>
+                                {privacyConfirm?.nextPrivate ? (
+                                    <>
+                                        <p>
+                                            The AI summary and any chat index will be removed. The file itself stays uploaded — you can still view and download it.
+                                        </p>
+                                        <ul className="list-disc list-inside text-xs text-muted-foreground space-y-0.5">
+                                            <li>No summarization, OCR, or embedding</li>
+                                            <li>Chat is disabled for this file</li>
+                                            <li>Only the filename is searchable</li>
+                                        </ul>
+                                    </>
+                                ) : (
+                                    <>
+                                        <p>
+                                            The file&apos;s contents will be re-extracted and sent to the AI for summarization, entity extraction, and semantic indexing.
+                                        </p>
+                                        <ul className="list-disc list-inside text-xs text-muted-foreground space-y-0.5">
+                                            <li>Summary and entities will be generated</li>
+                                            <li>Chat will become available</li>
+                                            <li>Content becomes searchable across the index</li>
+                                        </ul>
+                                    </>
+                                )}
+                            </div>
+                        </AlertDialogDescription>
+                    </AlertDialogHeader>
+                    <AlertDialogFooter>
+                        <AlertDialogCancel disabled={privacyBusy}>Cancel</AlertDialogCancel>
+                        <AlertDialogAction
+                            onClick={(e) => { e.preventDefault(); confirmTogglePrivacy(); }}
+                            disabled={privacyBusy}
+                            className={privacyConfirm?.nextPrivate ? "" : "bg-violet-600 hover:bg-violet-700 text-white"}
+                        >
+                            {privacyBusy
+                                ? "Updating…"
+                                : privacyConfirm?.nextPrivate ? "Mark private" : "Enable AI"}
+                        </AlertDialogAction>
+                    </AlertDialogFooter>
+                </AlertDialogContent>
+            </AlertDialog>
         </div >
     );
 }