Added fixes for search pipeline and frontend changes

backend/package.json

@@ -59,10 +59,11 @@
     "dotenv": "^16.5.0",
     "express": "^5.1.0",
     "express-rate-limit": "^7.5.1",
+    "gpt-tokenizer": "^3.4.0",
     "jsonwebtoken": "^9.0.2",
+    "mammoth": "^1.8.0",
     "mongodb": "^6.17.0",
     "mongoose": "^8.16.0",
-    "mammoth": "^1.8.0",
     "multer": "^2.0.0",
     "nodemailer": "^7.0.4",
     "pdf-parse": "^1.1.1",

backend/src/app.ts

@@ -14,6 +14,7 @@ import connectDB from "./db/mongo.js";
 import fileRouter from "./routes/file.js";
 import { requestContextMiddleware } from "./middleware/requestContext.js";
 import { csrfMiddleware } from "./middleware/csrf.js";
+import { warmupWeaviate, startWeaviateKeepalive } from "./db/weaviate_client.js";
 
 const app = express();
 const PORT = process.env.PORT || 4000;
@@ -38,6 +39,11 @@ app.use(csrfMiddleware);
 await connectDB()
 await setupPubSub();
 
+// Pre-warm Weaviate so the first user query doesn't pay TCP+TLS handshake cost.
+// Don't block startup if it fails — search will retry on the first real query.
+warmupWeaviate().catch(() => { /* logged inside */ });
+startWeaviateKeepalive();
+
 
 app.get("/", (req, res) => {
     res.send("SmartDrive backend running 🚀");

backend/src/db/weaviate_client.ts

@@ -1,29 +1,90 @@
-import weaviate from 'weaviate-client'
+import weaviate, { WeaviateClient } from 'weaviate-client';
 import logger from '../logger.js';
 
 const WEAVIATE_URL = process.env.WEAVIATE_URL as string;
 const WEAVIATE_API_KEY = process.env.WEAVIATE_API_KEY as string;
 
-const getWeaviateClient = async () => {
+// ---------- Singleton connection ----------
+// Previously, every call to getWeaviateClient() created a NEW connection,
+// adding 200-500ms TCP+TLS handshake to every search and chat query.
+// Now we reuse one client across the process lifetime.
 
+let _client: WeaviateClient | null = null;
+let _connecting: Promise<WeaviateClient | null> | null = null;
+
+const _connect = async (): Promise<WeaviateClient | null> => {
     if (!WEAVIATE_URL || !WEAVIATE_API_KEY) {
-        logger.error("WEAVIATE_CLUSTER_URL or WEAVIATE_API_KEY environment variables not set.")
-        return;
+        logger.error("WEAVIATE_URL or WEAVIATE_API_KEY environment variables not set.");
+        return null;
     }
     try {
-        const client = await weaviate.connectToWeaviateCloud(
-            WEAVIATE_URL, {
-            authCredentials: new weaviate.ApiKey(WEAVIATE_API_KEY),
-        }
-        )
-        logger.info("Successfully connected to Weaviate.")
-
-        return client
+        const c = await weaviate.connectToWeaviateCloud(
+            WEAVIATE_URL,
+            { authCredentials: new weaviate.ApiKey(WEAVIATE_API_KEY) },
+        );
+        logger.info("Successfully connected to Weaviate (cached for process lifetime).");
+        return c;
+    } catch (e) {
+        logger.error(`Failed to connect to Weaviate: ${e}`);
+        return null;
     }
-    catch (e) {
-        logger.error(`Failed to connect to Weaviate: ${e}`)
-        return
+};
+
+const getWeaviateClient = async (): Promise<WeaviateClient | null> => {
+    if (_client) return _client;
+
+    // If a connection is already being established, await the same promise
+    // instead of starting a second handshake (avoids thundering-herd on cold start).
+    if (_connecting) return _connecting;
+
+    _connecting = _connect().then((c) => {
+        _client = c;
+        _connecting = null;
+        return c;
+    });
+    return _connecting;
+};
+
+// ---------- Pre-warm on boot ----------
+// Called from app.ts at startup so the first request doesn't pay connection cost.
+
+export const warmupWeaviate = async (): Promise<void> => {
+    const start = Date.now();
+    const c = await getWeaviateClient();
+    if (c) {
+        try {
+            await c.isLive();
+            logger.info(`Weaviate warmup OK in ${Date.now() - start}ms`);
+        } catch (e) {
+            logger.warn(`Weaviate warmup failed: ${e}`);
+        }
     }
-}
+};
+
+// ---------- Keepalive ----------
+// Cloud Run keeps idle TCP for some time, but Weaviate Cloud may also drop
+// connections after periods of inactivity. A periodic isLive() ping keeps
+// the connection warm and lets us detect dropped connections early.
+
+const KEEPALIVE_MS = 5 * 60 * 1000; // 5 minutes
+let _keepaliveTimer: NodeJS.Timeout | null = null;
+
+export const startWeaviateKeepalive = (): void => {
+    if (_keepaliveTimer) return; // already running
+    _keepaliveTimer = setInterval(async () => {
+        try {
+            const c = await getWeaviateClient();
+            if (c) await c.isLive();
+        } catch (e) {
+            // If the keepalive fails, reset the client so the next real query
+            // re-establishes. Better to fail fast than serve from a dead conn.
+            logger.warn(`Weaviate keepalive failed, resetting client: ${e}`);
+            _client = null;
+        }
+    }, KEEPALIVE_MS);
+    // Don't keep the process alive just for this timer.
+    if (typeof _keepaliveTimer.unref === "function") _keepaliveTimer.unref();
+    logger.info(`Weaviate keepalive started (interval ${KEEPALIVE_MS / 1000}s)`);
+};
 
-export default getWeaviateClient;
+export default getWeaviateClient;

backend/src/handlers/fileHandler.ts

@@ -21,6 +21,14 @@ const capHistory = (history: ChatTurn[] | undefined): ChatTurn[] => {
     return valid.slice(-HISTORY_TURN_CAP);
 };
 
+// Personalization: bump access count + recency on file interaction. Fire-and-forget.
+const touchFileAccess = (fileId: string): void => {
+    UserFile.updateOne(
+        { _id: fileId },
+        { $inc: { accessCount: 1 }, $set: { lastAccessedAt: new Date() } },
+    ).catch((err) => logger.warn(`touchFileAccess fileId=${fileId} failed: ${err}`));
+};
+
 const getUserFile = (fileRecord: UserFileType | null) => {
     const filePath = `${fileRecord?.userId}/${fileRecord?.fileHash}`;
     return bucket.file(filePath);
@@ -90,6 +98,7 @@ const generateFileSignedUrl = async (req: AuthenticatedRequest, res: Response):
         }
 
         const [url] = await file.getSignedUrl(options);
+        touchFileAccess(fileId); // R6 personalization signal
         res.status(200).json({ url });
         return
 
@@ -269,6 +278,7 @@ const prepareChat = async (req: AuthenticatedRequest, res: Response): Promise<vo
             res.status(409).json({ message: result.reason });
             return;
         }
+        touchFileAccess(fileId); // R6 personalization signal
         res.status(200).json(result);
     } catch (err) {
         logger.error(`prepareChat failed for fileId=${fileId}:`, err);

backend/src/handlers/queryHandler.ts

@@ -1,7 +1,9 @@
 import { Response } from "express";
+import mongoose from "mongoose";
 import logger from "../logger.js";
 import { queryWeaviate } from "../services/queryWeaviate.js";
 import { AuthenticatedRequest } from "../middleware/auth.js";
+import SearchClick from "../models/searchClickModel.js";
 
 const queryHandler = async (req: AuthenticatedRequest, res: Response): Promise<void> => {
     const { userQuery, queryCollection } = req.query;
@@ -31,5 +33,45 @@ const queryHandler = async (req: AuthenticatedRequest, res: Response): Promise<v
 
 }
 
+// R10 — Click logging endpoint.
+// Frontend posts here when a user clicks a search result. Records
+// (userId, query, fileId, rank) for later CTR analysis and learned-rank
+// training. Fire-and-forget: we never block the user.
+const logSearchClick = async (req: AuthenticatedRequest, res: Response): Promise<void> => {
+    try {
+        const userId = req.user?._id;
+        if (!userId) {
+            res.status(401).json({ error: 'Not authorized.' });
+            return;
+        }
+        const { query, fileId, rank } = req.body as {
+            query?: string;
+            fileId?: string;
+            rank?: number;
+        };
+        if (!query || !fileId || typeof rank !== 'number' || rank < 1) {
+            res.status(400).json({ error: 'Missing query/fileId/rank.' });
+            return;
+        }
+        if (!mongoose.Types.ObjectId.isValid(fileId)) {
+            res.status(400).json({ error: 'Invalid fileId.' });
+            return;
+        }
+        const day = new Date().toISOString().slice(0, 10);
+        // Fire and forget — return 200 immediately, don't block on the write.
+        SearchClick.create({
+            userId,
+            query: query.trim().toLowerCase().slice(0, 500),
+            fileId: new mongoose.Types.ObjectId(fileId),
+            rank: Math.min(rank, 100),
+            day,
+        }).catch((err) => logger.warn(`searchClick insert failed: ${err}`));
+        res.status(200).json({ ok: true });
+    } catch (error) {
+        logger.error('logSearchClick failed:', error);
+        res.status(500).json({ error: 'Server Error' });
+    }
+};
 
+export { logSearchClick };
 export default queryHandler;

backend/src/handlers/uploadHandler.ts

@@ -162,6 +162,9 @@ const getUploads = async (req: AuthenticatedRequest, res: Response): Promise<voi
         summary: enrichment?.summary,
         index_json: enrichment?.indexJson,
         is_private: !!f.isPrivate,
+        // Live extraction progress for UI ("Summarizing with AI 2/4").
+        // Worker writes this; frontend's poll loop picks it up.
+        extraction_progress: status === 'processing' || status === 'pending' ? f.extractionProgress : undefined,
       };
     });
 

backend/src/models/searchClickModel.ts

@@ -0,0 +1,45 @@
+import mongoose from "mongoose";
+
+/**
+ * R10 — Click logging.
+ * Stores (user, query, clicked_file_id, rank) tuples so we can:
+ *   1. Compute CTR@1 (% of clicks at position 1) — cheap quality signal
+ *   2. Spot pathological queries (zero-click queries) — broken searches
+ *   3. Train a learned ranker once volume justifies it
+ *
+ * Lightweight: one document per click, indexed by (userId, day) for fast
+ * dashboards. TTL on createdAt to auto-prune older than 90 days.
+ */
+export interface SearchClickType extends mongoose.Document {
+    userId: mongoose.Types.ObjectId;
+    /** Normalized query string (trimmed + lowercased). */
+    query: string;
+    /** ID of the file the user clicked. */
+    fileId: mongoose.Types.ObjectId;
+    /** Position in the result list (1-based). */
+    rank: number;
+    /** Day in YYYY-MM-DD (UTC) for cheap aggregation. */
+    day: string;
+    createdAt: Date;
+}
+
+const schema = new mongoose.Schema(
+    {
+        userId: { type: mongoose.Schema.Types.ObjectId, ref: "User", required: true, index: true },
+        query: { type: String, required: true, maxlength: 500 },
+        fileId: { type: mongoose.Schema.Types.ObjectId, ref: "UserFile", required: true },
+        rank: { type: Number, required: true, min: 1, max: 100 },
+        day: { type: String, required: true, index: true },
+    },
+    {
+        timestamps: { createdAt: true, updatedAt: false },
+    }
+);
+
+// TTL: auto-delete clicks older than 90 days.
+schema.index({ createdAt: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
+// Common analytics query: clicks for a user, sorted by day desc.
+schema.index({ userId: 1, day: -1 });
+
+const SearchClick = mongoose.model<SearchClickType>("SearchClick", schema);
+export default SearchClick;

backend/src/models/userFileModel.ts

@@ -18,6 +18,13 @@ export interface UserFileType extends mongoose.Document {
      *  no body embedding, no chunks. Only filename + metadata are indexed so the
      *  user can still find the file by name. Chat is disabled for private files. */
     isPrivate?: boolean;
+    /** Personalization signal — files the user interacts with rank higher in
+     *  search. Touched on chat-prep, chat-stream, view, download. */
+    lastAccessedAt?: Date | null;
+    accessCount?: number;
+    /** Extraction progress for UI ("extracting page 3 of 12"). Worker
+     *  updates this periodically during processing. Cleared when status='done'. */
+    extractionProgress?: { current: number; total: number; stage: string } | null;
     createdAt: Date;
     updatedAt: Date;
 }
@@ -64,6 +71,13 @@ const userFileSchema = new mongoose.Schema(
             default: false,
             index: true,
         },
+        lastAccessedAt: { type: Date, default: null },
+        accessCount: { type: Number, default: 0 },
+        extractionProgress: {
+            current: { type: Number, default: 0 },
+            total: { type: Number, default: 0 },
+            stage: { type: String, default: "" },
+        },
     },
     {
         timestamps: true,

backend/src/routes/query.ts

@@ -1,9 +1,11 @@
 import {Router} from "express";
-import queryHandler from "../handlers/queryHandler.js";
+import queryHandler, { logSearchClick } from "../handlers/queryHandler.js";
 import { verifyToken } from "../middleware/auth.js";
 
 const queryRouter = Router();
 
 queryRouter.get('/', verifyToken, queryHandler);
+// R10 — click logging
+queryRouter.post('/click', verifyToken, logSearchClick);
 
 export default queryRouter;