Optimize performance

next.config.ts

@@ -14,6 +14,9 @@ const nextConfig: NextConfig = {
     "notebook.127.0.0.1.nip.io",
     "dashboard.127.0.0.1.nip.io",
   ],
+  turbopack: {
+    root: process.cwd(),
+  },
 };
 
 export default nextConfig;

package.json

@@ -24,7 +24,7 @@
     "@antv/chart-visualization-skills": "0.1.3",
     "@effect/platform": "^0.96.1",
     "@neondatabase/serverless": "^1.1.0",
-    "@ontos-ai/knowhere-sdk": "^0.6.0",
+    "@ontos-ai/knowhere-sdk": "^0.10.0",
     "@radix-ui/react-alert-dialog": "^1.1.15",
     "@radix-ui/react-checkbox": "^1.3.3",
     "@radix-ui/react-dialog": "^1.1.15",

src/app/api/sources/[sourceId]/chunks/route.test.ts

@@ -2,14 +2,19 @@ import { NextRequest } from "next/server"
 import { beforeEach, describe, expect, it, vi } from "vitest"
 
 const mocks = vi.hoisted(() => ({
+  blobGet: vi.fn(),
+  blobPut: vi.fn(),
+  deleteBlob: vi.fn(),
   ensureApiKeyForWorkspace: vi.fn(),
   ensureWorkspace: vi.fn(),
   fetchDemoChunkPage: vi.fn(),
   findSourceInWorkspace: vi.fn(),
   getCurrentUser: vi.fn(),
   getSourceParseAssetUrls: vi.fn(),
+  localizeRemoteDocument: vi.fn(),
   makeKnowhereClient: vi.fn(),
   requireUser: vi.fn(),
+  updateSourceRevisionKey: vi.fn(),
 }))
 
 vi.mock("next/headers", () => ({
@@ -36,10 +41,18 @@ vi.mock("@/integrations/knowhere", () => ({
   makeKnowhereClient: mocks.makeKnowhereClient,
 }))
 
+vi.mock("@vercel/blob", () => ({
+  del: mocks.deleteBlob,
+  get: mocks.blobGet,
+  put: mocks.blobPut,
+}))
+
 vi.mock("@/domains/sources/service", () => ({
   sourceService: {
     findInWorkspace: mocks.findSourceInWorkspace,
     getParseAssetUrls: mocks.getSourceParseAssetUrls,
+    localizeRemoteDocument: mocks.localizeRemoteDocument,
+    updateSourceRevisionKey: mocks.updateSourceRevisionKey,
   },
 }))
 
@@ -54,6 +67,11 @@ import { GET } from "./route"
 describe("GET /api/sources/[sourceId]/chunks", () => {
   beforeEach(() => {
     vi.clearAllMocks()
+    mocks.blobGet.mockResolvedValue(null)
+    mocks.blobPut.mockImplementation(async (pathname: string) => ({
+      url: `https://blob.example/${pathname}`,
+    }))
+    mocks.updateSourceRevisionKey.mockResolvedValue(null)
   })
 
   it("serves API-owned demo chunks for anonymous canonical demo sources", async () => {
@@ -502,7 +520,48 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
     })
   })
 
-  it("does not load chunks from unlocalized remote source ids", async () => {
+  it("materializes a remote source id on open before loading chunks", async () => {
+    const knowhereClient = {
+      documents: {
+        list: vi.fn(async () => ({
+          documents: [
+            {
+              documentId: "doc_remote",
+              namespace: "default",
+              status: "active",
+              currentJobResultId: "job_result_1",
+              sourceFileName: "remote.pdf",
+              documentMetadata: {
+                mimeType: "application/pdf",
+              },
+            },
+          ],
+        })),
+        listChunks: vi.fn(async () => ({
+          documentId: "doc_remote",
+          jobResultId: "job_result_1",
+          chunks: [
+            {
+              id: "dchk_remote",
+              chunkId: "parser_remote",
+              chunkType: "text",
+              content: "Remote chunk",
+              sectionPath: "Summary",
+              sourceChunkPath: "Default_Root/remote.pdf/Summary",
+              filePath: null,
+              metadata: {},
+              sortOrder: 0,
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 1,
+            total: 1,
+            totalPages: 1,
+          },
+        })),
+      },
+    }
     mocks.getCurrentUser.mockResolvedValue({
       id: "user_1",
       email: null,
@@ -515,6 +574,27 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
       createdAt: new Date("2026-05-10T00:00:00.000Z"),
     })
     mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo"))
+    mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123")
+    mocks.makeKnowhereClient.mockReturnValue(knowhereClient)
+    mocks.localizeRemoteDocument.mockResolvedValue({
+      id: "00000000-0000-0000-0000-000000000009",
+      workspaceId: "workspace_1",
+      title: "remote.pdf",
+      mimeType: "application/pdf",
+      sizeBytes: 0,
+      status: "ready",
+      failureReason: null,
+      knowhereJobId: "job_result_1",
+      knowhereDocumentId: "doc_remote",
+      stagedBlobPathname: null,
+      stagedBlobUrl: null,
+      originalBlobPathname: null,
+      originalBlobUrl: null,
+      demoKey: null,
+      createdAt: new Date("2026-05-10T00:00:00.000Z"),
+      updatedAt: new Date("2026-05-10T00:00:00.000Z"),
+      deletedAt: null,
+    })
 
     const response = await GET(
       new NextRequest(
@@ -527,12 +607,46 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
       },
     )
 
-    await expect(response.json()).resolves.toEqual({
-      message: "Source not found.",
+    await expect(response.json()).resolves.toMatchObject({
+      chunks: [
+        {
+          chunkId: "dchk_remote",
+          parserChunkId: "parser_remote",
+          documentId: "doc_remote",
+          sourceTitle: "remote.pdf",
+        },
+      ],
+      pagination: {
+        page: 1,
+        pageSize: 1,
+        total: 1,
+      },
     })
-    expect(response.status).toBe(404)
+    expect(response.status).toBe(200)
     expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled()
-    expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled()
-    expect(mocks.makeKnowhereClient).not.toHaveBeenCalled()
+    expect(mocks.ensureApiKeyForWorkspace).toHaveBeenCalledWith(
+      "workspace_1",
+      "session=abc",
+    )
+    expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith(
+      "workspace_1",
+      {
+        documentId: "doc_remote",
+        namespace: "default",
+        status: "ready",
+        title: "remote.pdf",
+        mimeType: "application/pdf",
+        sizeBytes: undefined,
+        revisionKey: "job_result_1",
+      },
+    )
+    expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith(
+      "doc_remote",
+      {
+        page: 1,
+        pageSize: 1,
+        includeAssetUrls: true,
+      },
+    )
   })
 })

src/app/api/sources/[sourceId]/route.test.ts

@@ -14,7 +14,9 @@ const mocks = vi.hoisted(() => {
     hideDemoSource: vi.fn(),
     makeKnowhereClient: vi.fn(),
     requireUser: vi.fn(),
+    retrySourceToKnowhere: vi.fn(),
     softDeleteSource: vi.fn(),
+    startBackgroundReconciliation: vi.fn(),
   };
 });
 
@@ -46,10 +48,15 @@ vi.mock("@/integrations/knowhere", () => ({
   makeKnowhereClient: mocks.makeKnowhereClient,
 }));
 
+vi.mock("@/domains/sources/background-reconcile", () => ({
+  startBackgroundReconciliation: mocks.startBackgroundReconciliation,
+}));
+
 vi.mock("@/domains/sources/service", () => ({
   sourceService: {
     findInWorkspace: mocks.findSourceInWorkspace,
     hideDemoSource: mocks.hideDemoSource,
+    retrySourceToKnowhere: mocks.retrySourceToKnowhere,
     softDelete: mocks.softDeleteSource,
   },
 }));
@@ -255,4 +262,124 @@ describe("PATCH /api/sources/[sourceId]", () => {
     expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled();
     expect(mocks.archive).not.toHaveBeenCalled();
   });
+
+  it("retries a failed source and starts background reconciliation", async () => {
+    mocks.requireUser.mockResolvedValue({ id: "user_1" });
+    mocks.ensureWorkspace.mockResolvedValue({ id: "workspace_1" });
+    mocks.findSourceInWorkspace.mockResolvedValue({
+      id: "source_1",
+      workspaceId: "workspace_1",
+      title: "lecture.pdf",
+      mimeType: "application/pdf",
+      sizeBytes: 5,
+      status: "failed",
+      failureReason: "Knowhere upload failed.",
+      knowhereJobId: null,
+      knowhereDocumentId: null,
+      stagedBlobPathname: null,
+      stagedBlobUrl: null,
+      originalBlobPathname: "source-uploads/upload_1/document.pdf",
+      originalBlobUrl:
+        "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf",
+      demoKey: null,
+      createdAt: new Date("2026-05-10T00:00:00Z"),
+      updatedAt: new Date("2026-05-10T00:00:00Z"),
+      deletedAt: null,
+    });
+    mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123");
+    const knowhereClient = {
+      jobs: {
+        create: vi.fn(),
+        get: vi.fn(),
+        upload: vi.fn(),
+      },
+      documents: {
+        archive: mocks.archive,
+      },
+    };
+    mocks.makeKnowhereClient.mockReturnValue(knowhereClient);
+    mocks.retrySourceToKnowhere.mockResolvedValue({
+      id: "source_1",
+      workspaceId: "workspace_1",
+      title: "lecture.pdf",
+      mimeType: "application/pdf",
+      sizeBytes: 5,
+      status: "parsing",
+      failureReason: null,
+      knowhereJobId: "job_retry",
+      knowhereDocumentId: null,
+      stagedBlobPathname: null,
+      stagedBlobUrl: null,
+      originalBlobPathname: "source-uploads/upload_1/document.pdf",
+      originalBlobUrl:
+        "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf",
+      demoKey: null,
+      createdAt: new Date("2026-05-10T00:00:00Z"),
+      updatedAt: new Date("2026-05-10T00:00:00Z"),
+      deletedAt: null,
+    });
+    mocks.startBackgroundReconciliation.mockResolvedValue(undefined);
+
+    const response = await PATCH(
+      new NextRequest("http://localhost:3001/api/sources/source_1", {
+        method: "PATCH",
+        body: JSON.stringify({ retry: true }),
+      }),
+      { params: Promise.resolve({ sourceId: "source_1" }) },
+    );
+
+    await expect(response.json()).resolves.toEqual({
+      source: {
+        id: "source_1",
+        kind: "workspace",
+        title: "lecture.pdf",
+        mimeType: "application/pdf",
+        status: "parsing",
+        documentId: undefined,
+        originalFile: {
+          url: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf",
+          mimeType: "application/pdf",
+          sizeBytes: 5,
+        },
+      },
+    });
+    expect(response.status).toBe(200);
+    expect(mocks.retrySourceToKnowhere).toHaveBeenCalledWith(
+      { id: "workspace_1" },
+      expect.objectContaining({ id: "source_1", status: "failed" }),
+      knowhereClient,
+    );
+    expect(mocks.startBackgroundReconciliation).toHaveBeenCalledWith(
+      "workspace_1",
+      "source_1",
+      "jwt_123",
+    );
+  });
+
+  it("rejects retry requests for failed rows without a saved original Blob", async () => {
+    mocks.requireUser.mockResolvedValue({ id: "user_1" });
+    mocks.ensureWorkspace.mockResolvedValue({ id: "workspace_1" });
+    mocks.findSourceInWorkspace.mockResolvedValue({
+      id: "source_1",
+      status: "failed",
+      originalBlobPathname: null,
+      originalBlobUrl: null,
+    });
+
+    const response = await PATCH(
+      new NextRequest("http://localhost:3001/api/sources/source_1", {
+        method: "PATCH",
+        body: JSON.stringify({ retry: true }),
+      }),
+      { params: Promise.resolve({ sourceId: "source_1" }) },
+    );
+
+    await expect(response.json()).resolves.toEqual({
+      message:
+        "This source cannot be retried because its original file is unavailable.",
+    });
+    expect(response.status).toBe(409);
+    expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled();
+    expect(mocks.retrySourceToKnowhere).not.toHaveBeenCalled();
+  });
 });