diff --git a/backend/src/handlers/fileHandler.ts b/backend/src/handlers/fileHandler.ts index 999beeb..204e901 100644 --- a/backend/src/handlers/fileHandler.ts +++ b/backend/src/handlers/fileHandler.ts @@ -194,6 +194,24 @@ const triggerExtraction = async (req: AuthenticatedRequest, res: Response): Prom return; } + // Hard guard: a private file must NEVER be sent to the worker for + // LLM extraction. The worker should also check isPrivate and write a + // stub, but a stale worker deploy could ignore the flag — so refuse + // here as well. Re-extraction of a private file is a no-op; the + // filename stub is already indexed. + if (fileRecord.isPrivate) { + logger.info(`triggerExtraction skipped for private fileId=${fileId} — already indexed as stub`); + fileRecord.extractionStatus = 'done'; + fileRecord.extractionError = undefined; + fileRecord.chatReady = false; + await fileRecord.save(); + res.status(200).json({ + message: 'File is marked private — re-extraction would send contents to AI and is blocked. Toggle privacy off first.', + extraction_status: 'done', + }); + return; + } + // Reset state so the UI immediately reflects "queued" — even before // the worker picks the message up. Also flush the cached chat index: // a re-extracted file should be chunked from the new text, not the old. diff --git a/backend/src/services/searchPipeline.ts b/backend/src/services/searchPipeline.ts index 9e09906..4e8f9b4 100644 --- a/backend/src/services/searchPipeline.ts +++ b/backend/src/services/searchPipeline.ts @@ -152,6 +152,16 @@ const recencyBoost = (createdAt: unknown, baseScore: number): number => { type WClient = Awaited>; +/** Pick alpha based on query length. Short queries (1-3 tokens) are usually + * keyword-driven — bias toward BM25. Longer queries are natural language — + * let dense pull more weight. */ +const pickAlpha = (query: string): number => { + const tokens = query.trim().split(/\s+/).filter(Boolean); + if (tokens.length <= 3) return 0.25; // 75% BM25, 25% dense + if (tokens.length <= 6) return 0.5; // balanced + return 0.7; // 70% dense, 30% BM25 +}; + const chunkHybridSignal = async ( client: NonNullable, chunksName: string, @@ -163,7 +173,7 @@ const chunkHybridSignal = async ( const col = client.collections.get(chunksName); const res = await col.query.hybrid(query, { vector, - alpha: 0.5, + alpha: pickAlpha(query), limit: 50, filters: col.filter.byProperty("user_id").equal(userId), returnMetadata: ["score"], @@ -180,6 +190,40 @@ const chunkHybridSignal = async ( return [...seen.values()]; }; +/** Pure BM25 over chunk text. Critical for rare-term queries like proper nouns + * ("Guwahati", "Acme Corp") — hybrid's dense half dilutes exact matches, + * but pure BM25 ranks the unique-term file at #1 reliably. RRF fuses this + * with the hybrid signal so common-word queries still benefit from semantic. */ +const chunkBM25Signal = async ( + client: NonNullable, + chunksName: string, + userId: string, + query: string, +): Promise => { + if (!(await client.collections.exists(chunksName))) return []; + const col = client.collections.get(chunksName); + try { + const res = await col.query.bm25(query, { + queryProperties: ["chunk_text"], + limit: 50, + filters: col.filter.byProperty("user_id").equal(userId), + returnMetadata: ["score"], + }); + const seen = new Map(); + let rank = 1; + for (const obj of res.objects) { + const p = obj.properties as { file_id?: string; chunk_text?: string }; + const fid = String(p.file_id ?? ""); + if (!fid || seen.has(fid)) { rank++; continue; } + seen.set(fid, { file_id: fid, rank: rank++, matched_chunk: p.chunk_text }); + } + return [...seen.values()]; + } catch (e) { + logger.warn(`chunkBM25Signal on ${chunksName} failed: ${e}`); + return []; + } +}; + const filenameBM25Signal = async ( client: NonNullable, summaryName: string, @@ -294,12 +338,42 @@ const entitySignal = async ( // ---------- 6. Orchestrator ---------- -const intentWeights = (intent: Intent): { chunk: number; filename: number; summary: number; entity: number } => { +const intentWeights = (intent: Intent): { chunk: number; chunkBM25: number; filename: number; summary: number; entity: number } => { // Defaults are balanced. Filename intent boosts filename + summary (where the name lives). // Pure content intent boosts chunk/body. - if (intent.filename) return { chunk: 1.0, filename: 2.5, summary: 1.5, entity: 1.5 }; - if (intent.exactPhrases.length > 0) return { chunk: 2.0, filename: 1.0, summary: 1.5, entity: 2.0 }; - return { chunk: 1.5, filename: 1.0, summary: 1.0, entity: 1.5 }; + // chunkBM25 is the pure-lexical companion to chunk hybrid — weight it strongly + // because exact-term matches (proper nouns, IDs, rare words) are what users + // most often expect to win. + if (intent.filename) return { chunk: 1.0, chunkBM25: 1.5, filename: 2.5, summary: 1.5, entity: 1.5 }; + if (intent.exactPhrases.length > 0) return { chunk: 1.5, chunkBM25: 2.5, filename: 1.0, summary: 1.5, entity: 2.0 }; + return { chunk: 1.5, chunkBM25: 2.0, filename: 1.0, summary: 1.0, entity: 1.5 }; +}; + +/** Exact-substring match post-RRF bonus. If the query token literally appears + * in the file's filename, summary, or matched chunk, give a meaningful score + * bump. Catches the "one file uniquely contains this term — it should be #1" + * failure mode that RRF alone misses. + * + * Scaled to ~0.05 per match, which dominates the typical RRF score + * (range 0.01-0.10) without overwhelming intentional ranking. */ +const exactMatchBoost = ( + query: string, + properties: Record, + matchedChunk?: string, +): number => { + const tokens = query.toLowerCase().split(/\s+/).filter((t) => t.length >= 3); + if (tokens.length === 0) return 0; + const filename = String(properties.filename ?? "").toLowerCase(); + const summary = String(properties.summary ?? "").toLowerCase(); + const chunk = (matchedChunk ?? "").toLowerCase(); + let bonus = 0; + for (const tok of tokens) { + // Filename match is the strongest signal — user named the file with this word. + if (filename.includes(tok)) bonus += 0.08; + if (summary.includes(tok)) bonus += 0.04; + if (chunk.includes(tok)) bonus += 0.03; + } + return bonus; }; export const runSearchPipeline = async ( @@ -337,6 +411,8 @@ export const runSearchPipeline = async ( await Promise.all(pairs.flatMap(({ summary: summaryName, chunks: chunksName }) => [ chunkHybridSignal(client, chunksName, userId, queryForRetrieval, queryVector) .then((hits) => allSignals.push({ name: "content", hits, weight: weights.chunk })), + chunkBM25Signal(client, chunksName, userId, queryForRetrieval) + .then((hits) => allSignals.push({ name: "content-bm25", hits, weight: weights.chunkBM25 })), filenameBM25Signal(client, summaryName, userId, queryForRetrieval) .then((hits) => allSignals.push({ name: "filename", hits, weight: weights.filename })), summaryHybridSignal(client, summaryName, userId, queryForRetrieval, queryVector) @@ -379,17 +455,21 @@ export const runSearchPipeline = async ( } })); - // Final scoring: fused RRF score + recency boost (if intent suggests it). + // Final scoring: fused RRF + recency boost + exact-match boost. + // The exact-match boost is the critical safety net for rare-term queries + // ("Guwahati", "INV-12345"). Without it, RRF can demote the one file that + // uniquely contains the term in favor of files that rank middling-but-everywhere. const recencyMultiplier = analysis.intent.recency ? 3 : 1; const results: SearchResult[] = fileIds.map((fid) => { const f = fused.get(fid)!; const parent = summariesByFile.get(fid) ?? {}; - const boost = recencyMultiplier * recencyBoost(parent.created_at, f.score); + const recBoost = recencyMultiplier * recencyBoost(parent.created_at, f.score); + const exactBoost = exactMatchBoost(queryForRetrieval, parent, f.bestChunk); const entityMatch = entityMatchByFile.get(fid); return { ...parent, file_id: fid, - score: f.score + boost, + score: f.score + recBoost + exactBoost, matched_in: [...f.matched_in], matched_chunk: f.bestChunk, matched_entities: entityMatch?.entities.length ? entityMatch.entities : undefined, diff --git a/smartdrive-extractor/utils/ingestion_router.py b/smartdrive-extractor/utils/ingestion_router.py index 861f0b9..537dce8 100644 --- a/smartdrive-extractor/utils/ingestion_router.py +++ b/smartdrive-extractor/utils/ingestion_router.py @@ -6,6 +6,7 @@ from smartdrive_core.mongo_status import update_status from .image_pipeline import process_image from .document_extractor import process_document +from .weaviate_utils import save_doc_private, save_image_private logger = logging.getLogger(__name__) @@ -38,6 +39,17 @@ def route_and_process(file_path: str, data: dict): update_status(str(file_id), "processing") + # Hard guard: a private file must never reach the LLM/OCR pipeline. + # Even if the per-processor check were missed, the router catches it. + if data.get("isPrivate"): + logger.info(f"Router: private file '{filename}' — writing filename stub only, skipping LLM/OCR") + if mime.startswith("image/"): + save_image_private(data) + else: + save_doc_private(data) + update_status(str(file_id), "done") + return {"message": f"Saved {filename} as private (content not indexed)", "created": True} + if mime.startswith("image/"): logger.info("checking if image exists") if check_file_exists(IMG_COLLECTION, str(file_id), str(user_id)): diff --git a/smartdrive-frontend/src/components/FileListWithDrawer.tsx b/smartdrive-frontend/src/components/FileListWithDrawer.tsx index 2ae887e..b350c8c 100644 --- a/smartdrive-frontend/src/components/FileListWithDrawer.tsx +++ b/smartdrive-frontend/src/components/FileListWithDrawer.tsx @@ -752,6 +752,8 @@ export function FileListWithDrawer({ const [drawerOpen, setDrawerOpen] = useState(false); const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false); + const [privacyConfirm, setPrivacyConfirm] = useState<{ file: UploadItem; nextPrivate: boolean } | null>(null); + const [privacyBusy, setPrivacyBusy] = useState(false); const [actionById, setActionById] = useState>({}); const [previewUrl, setPreviewUrl] = useState(null); @@ -907,21 +909,25 @@ export function FileListWithDrawer({ onRefresh(); }, [bulkSelected, onRefresh]); - const handleTogglePrivacy = useCallback(async (file: UploadItem) => { - const nextPrivate = !file.is_private; - const verb = nextPrivate ? "Mark as private" : "Enable AI features"; - const detail = nextPrivate - ? "This will remove the AI summary and disable chat for this file. The file itself stays uploaded." - : "This will re-extract the file and send its contents to the AI for summarization and indexing."; - if (!window.confirm(`${verb}?\n\n${detail}\n\nContinue?`)) return; + const requestTogglePrivacy = useCallback((file: UploadItem) => { + setPrivacyConfirm({ file, nextPrivate: !file.is_private }); + }, []); + + const confirmTogglePrivacy = useCallback(async () => { + if (!privacyConfirm) return; + const { file, nextPrivate } = privacyConfirm; + setPrivacyBusy(true); try { await apiClient.patch(`/file/${file.file_id}/privacy`, { isPrivate: nextPrivate }); toast.success(nextPrivate ? "Marked as private." : "AI features enabled. Re-extracting…"); + setPrivacyConfirm(null); onRefresh(); } catch (err) { toast.error(`Could not update privacy: ${(err as Error).message || "unknown error"}`); + } finally { + setPrivacyBusy(false); } - }, [onRefresh]); + }, [privacyConfirm, onRefresh]); const handleRetryAllFailed = useCallback(async () => { const failedIds = files @@ -1192,7 +1198,7 @@ export function FileListWithDrawer({ onDownload={() => handleDownload(file)} onExtract={() => handleExtract(file)} onToggleSelect={() => toggleSelect(file.file_id)} - onTogglePrivacy={() => handleTogglePrivacy(file)} + onTogglePrivacy={() => requestTogglePrivacy(file)} onDeleteRequest={() => { setSelectedId(file.file_id); setConfirmDeleteOpen(true); @@ -1456,6 +1462,67 @@ export function FileListWithDrawer({ + + { if (!open && !privacyBusy) setPrivacyConfirm(null); }} + > + + + + {privacyConfirm?.nextPrivate ? ( + <> Mark as private? + ) : ( + <> Enable AI features? + )} + + +
+

+ + {privacyConfirm?.file.filename ?? "This file"} + +

+ {privacyConfirm?.nextPrivate ? ( + <> +

+ The AI summary and any chat index will be removed. The file itself stays uploaded — you can still view and download it. +

+
    +
  • No summarization, OCR, or embedding
    • +
  • Chat is disabled for this file
    • +
  • Only the filename is searchable
    • +
+ + ) : ( + <> +

+ The file's contents will be re-extracted and sent to the AI for summarization, entity extraction, and semantic indexing. +

+
    +
  • Summary and entities will be generated
    • +
  • Chat will become available
    • +
  • Content becomes searchable across the index
    • +
+ + )} +
+ + + + Cancel + { e.preventDefault(); confirmTogglePrivacy(); }} + disabled={privacyBusy} + className={privacyConfirm?.nextPrivate ? "" : "bg-violet-600 hover:bg-violet-700 text-white"} + > + {privacyBusy + ? "Updating…" + : privacyConfirm?.nextPrivate ? "Mark private" : "Enable AI"} + + + + ); } \ No newline at end of file