Merge pull request #6244 from simstudioai/feat/v2-knowledge-multipart-upload

feat(knowledge): multipart knowledge document uploads

Author: TheodoreSpeaks

apps/docs/openapi-v2-knowledge.json

@@ -32,6 +32,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
       uploadId: parsed.data.params.uploadId,
       workspaceId,
       userId: user,
+      purpose: 'workspace_file',
       uploadToken: parsed.data.headers['upload-token'],
     })
     const metadata = upload.metadata as { folderId?: string | null }

apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts

@@ -29,6 +29,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
       uploadId: parsed.data.params.uploadId,
       workspaceId,
       userId: user,
+      purpose: 'workspace_file',
       uploadToken: parsed.data.headers['upload-token'],
     })
     const parts = await createUploadPartUrls({

apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts

@@ -27,6 +27,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Upl
       uploadId: parsed.data.params.uploadId,
       workspaceId,
       userId: user,
+      purpose: 'workspace_file',
       uploadToken: parsed.data.headers['upload-token'],
     })
     return NextResponse.json({ data: toV2FileUpload(await abortUploadSession(upload), null) })

apps/sim/app/api/files/uploads/[uploadId]/route.ts

@@ -0,0 +1,74 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { completeKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
+import { parseRequest } from '@/lib/api/server'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { completeUploadSession } from '@/lib/uploads/multipart-session/service'
+import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
+import {
+  requireKnowledgeDocumentUploadAccess,
+  requireKnowledgeDocumentUploadActor,
+  resolveKnowledgeDocumentUploadAttribution,
+} from '@/app/api/knowledge/[id]/documents/uploads/utils'
+import {
+  finalizeKnowledgeDocumentUpload,
+  getOwnedKnowledgeDocumentUpload,
+  toV2KnowledgeDocumentUpload,
+} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
+
+interface KnowledgeDocumentUploadRouteParams {
+  params: Promise<{ id: string; uploadId: string }>
+}
+
+export const POST = withRouteHandler(
+  async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
+    const actor = await requireKnowledgeDocumentUploadActor()
+    if (actor instanceof NextResponse) return actor
+    const parsed = await parseRequest(completeKnowledgeDocumentUploadContract, request, context)
+    if (!parsed.success) return parsed.response
+    const { id: knowledgeBaseId, uploadId } = parsed.data.params
+    const { workspaceId } = parsed.data.query
+    const access = await requireKnowledgeDocumentUploadAccess({
+      knowledgeBaseId,
+      workspaceId,
+      userId: actor.id,
+    })
+    if (access instanceof NextResponse) return access
+    const requestId = generateRequestId()
+    try {
+      const upload = getOwnedKnowledgeDocumentUpload({
+        knowledgeBaseId,
+        uploadId,
+        workspaceId,
+        userId: actor.id,
+        uploadToken: parsed.data.headers['upload-token'],
+      })
+      const completed = await completeUploadSession({
+        session: upload,
+        parts: parsed.data.body.parts,
+        finalize: (claimed) =>
+          finalizeKnowledgeDocumentUpload({
+            claimed,
+            knowledgeBaseId,
+            knowledgeBaseName: access.knowledgeBase.name,
+            workspaceId,
+            userId: actor.id,
+            resolveAttribution: () =>
+              resolveKnowledgeDocumentUploadAttribution({ workspaceId, userId: actor.id }),
+            source: 'ui',
+            requestId,
+            request,
+            actorName: actor.name,
+            actorEmail: actor.email,
+          }),
+      })
+      return NextResponse.json({
+        data: toV2KnowledgeDocumentUpload(completed.session, completed.value),
+      })
+    } catch (error) {
+      const classified = uploadSessionErrorResponse(error)
+      if (classified) return classified
+      throw error
+    }
+  }
+)

apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts

@@ -0,0 +1,55 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { createKnowledgeDocumentUploadPartUrlsContract } from '@/lib/api/contracts/knowledge/upload-sessions'
+import { parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service'
+import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
+import {
+  requireKnowledgeDocumentUploadAccess,
+  requireKnowledgeDocumentUploadActor,
+} from '@/app/api/knowledge/[id]/documents/uploads/utils'
+import { getOwnedKnowledgeDocumentUpload } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
+
+interface KnowledgeDocumentUploadRouteParams {
+  params: Promise<{ id: string; uploadId: string }>
+}
+
+export const POST = withRouteHandler(
+  async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
+    const actor = await requireKnowledgeDocumentUploadActor()
+    if (actor instanceof NextResponse) return actor
+    const parsed = await parseRequest(
+      createKnowledgeDocumentUploadPartUrlsContract,
+      request,
+      context
+    )
+    if (!parsed.success) return parsed.response
+    const { id: knowledgeBaseId, uploadId } = parsed.data.params
+    const { workspaceId } = parsed.data.query
+    const access = await requireKnowledgeDocumentUploadAccess({
+      knowledgeBaseId,
+      workspaceId,
+      userId: actor.id,
+    })
+    if (access instanceof NextResponse) return access
+    try {
+      const upload = getOwnedKnowledgeDocumentUpload({
+        knowledgeBaseId,
+        uploadId,
+        workspaceId,
+        userId: actor.id,
+        uploadToken: parsed.data.headers['upload-token'],
+      })
+      const parts = await createUploadPartUrls({
+        session: upload,
+        partNumbers: parsed.data.body.partNumbers,
+        localOrigin: request.nextUrl.origin,
+      })
+      return NextResponse.json({ data: { parts } })
+    } catch (error) {
+      const classified = uploadSessionErrorResponse(error)
+      if (classified) return classified
+      throw error
+    }
+  }
+)

apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts

@@ -0,0 +1,50 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { abortKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
+import { parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
+import {
+  requireKnowledgeDocumentUploadAccess,
+  requireKnowledgeDocumentUploadActor,
+} from '@/app/api/knowledge/[id]/documents/uploads/utils'
+import {
+  abortKnowledgeDocumentUpload,
+  getOwnedKnowledgeDocumentUpload,
+  toV2KnowledgeDocumentUpload,
+} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
+
+interface KnowledgeDocumentUploadRouteParams {
+  params: Promise<{ id: string; uploadId: string }>
+}
+
+export const DELETE = withRouteHandler(
+  async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
+    const actor = await requireKnowledgeDocumentUploadActor()
+    if (actor instanceof NextResponse) return actor
+    const parsed = await parseRequest(abortKnowledgeDocumentUploadContract, request, context)
+    if (!parsed.success) return parsed.response
+    const { id: knowledgeBaseId, uploadId } = parsed.data.params
+    const { workspaceId } = parsed.data.query
+    const access = await requireKnowledgeDocumentUploadAccess({
+      knowledgeBaseId,
+      workspaceId,
+      userId: actor.id,
+    })
+    if (access instanceof NextResponse) return access
+    try {
+      const upload = getOwnedKnowledgeDocumentUpload({
+        knowledgeBaseId,
+        uploadId,
+        workspaceId,
+        userId: actor.id,
+        uploadToken: parsed.data.headers['upload-token'],
+      })
+      const aborted = await abortKnowledgeDocumentUpload(upload, knowledgeBaseId)
+      return NextResponse.json({ data: toV2KnowledgeDocumentUpload(aborted, null) })
+    } catch (error) {
+      const classified = uploadSessionErrorResponse(error)
+      if (classified) return classified
+      throw error
+    }
+  }
+)

apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts

@@ -0,0 +1,118 @@
+/**
+ * @vitest-environment node
+ */
+import { NextRequest, NextResponse } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockCreateKnowledgeDocumentUploadSession,
+  mockRequireKnowledgeDocumentUploadAccess,
+  mockRequireKnowledgeDocumentUploadActor,
+  mockRequireKnowledgeDocumentUploadBilling,
+} = vi.hoisted(() => ({
+  mockCreateKnowledgeDocumentUploadSession: vi.fn(),
+  mockRequireKnowledgeDocumentUploadAccess: vi.fn(),
+  mockRequireKnowledgeDocumentUploadActor: vi.fn(),
+  mockRequireKnowledgeDocumentUploadBilling: vi.fn(),
+}))
+
+vi.mock('@/app/api/knowledge/[id]/documents/uploads/utils', () => ({
+  requireKnowledgeDocumentUploadAccess: mockRequireKnowledgeDocumentUploadAccess,
+  requireKnowledgeDocumentUploadActor: mockRequireKnowledgeDocumentUploadActor,
+  requireKnowledgeDocumentUploadBilling: mockRequireKnowledgeDocumentUploadBilling,
+}))
+vi.mock('@/app/api/files/uploads/utils', () => ({ uploadSessionErrorResponse: vi.fn() }))
+vi.mock('@/app/api/v2/knowledge/[id]/documents/uploads/utils', () => ({
+  createKnowledgeDocumentUploadSession: mockCreateKnowledgeDocumentUploadSession,
+  toV2KnowledgeDocumentUpload: (session: Record<string, unknown>) => ({
+    ...session,
+    name: session.fileName,
+    contentType: session.contentType,
+    size: session.fileSize,
+    expiresAt: '2026-08-05T00:00:00.000Z',
+    document: null,
+  }),
+}))
+
+import { POST } from '@/app/api/knowledge/[id]/documents/uploads/route'
+
+const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8'
+
+function request() {
+  return POST(
+    new NextRequest('http://localhost:3000/api/knowledge/kb-1/documents/uploads', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        workspaceId: WORKSPACE_ID,
+        name: 'guide.pdf',
+        contentType: 'application/pdf',
+        size: 1024,
+        tag1: 'product',
+        processingOptions: { recipe: 'default', lang: 'en' },
+      }),
+    }),
+    { params: Promise.resolve({ id: 'kb-1' }) }
+  )
+}
+
+describe('POST /api/knowledge/[id]/documents/uploads', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockRequireKnowledgeDocumentUploadActor.mockResolvedValue({ id: 'user-1' })
+    mockRequireKnowledgeDocumentUploadAccess.mockResolvedValue({
+      knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: WORKSPACE_ID },
+    })
+    mockRequireKnowledgeDocumentUploadBilling.mockResolvedValue({ actorUserId: 'user-1' })
+    mockCreateKnowledgeDocumentUploadSession.mockResolvedValue({
+      id: 'upload-1',
+      knowledgeBaseId: 'kb-1',
+      status: 'uploading',
+      fileName: 'guide.pdf',
+      contentType: 'application/pdf',
+      fileSize: 1024,
+      partSize: 8 * 1024 * 1024,
+      partCount: 1,
+      uploadToken: 'token',
+      error: null,
+    })
+  })
+
+  it('authorizes and bills before allocating a first-party upload session', async () => {
+    const response = await request()
+
+    expect(response.status).toBe(201)
+    expect(mockRequireKnowledgeDocumentUploadAccess).toHaveBeenCalledWith({
+      knowledgeBaseId: 'kb-1',
+      workspaceId: WORKSPACE_ID,
+      userId: 'user-1',
+    })
+    expect(mockCreateKnowledgeDocumentUploadSession).toHaveBeenCalledWith({
+      workspaceId: WORKSPACE_ID,
+      userId: 'user-1',
+      knowledgeBaseId: 'kb-1',
+      fileName: 'guide.pdf',
+      contentType: 'application/pdf',
+      fileSize: 1024,
+      metadata: {
+        tag1: 'product',
+        processingOptions: { recipe: 'default', lang: 'en' },
+      },
+    })
+    expect(mockRequireKnowledgeDocumentUploadBilling.mock.invocationCallOrder[0]).toBeLessThan(
+      mockCreateKnowledgeDocumentUploadSession.mock.invocationCallOrder[0]
+    )
+  })
+
+  it('does not bill or allocate storage when write access is denied', async () => {
+    mockRequireKnowledgeDocumentUploadAccess.mockResolvedValue(
+      NextResponse.json({ error: 'Forbidden' }, { status: 403 })
+    )
+
+    const response = await request()
+
+    expect(response.status).toBe(403)
+    expect(mockRequireKnowledgeDocumentUploadBilling).not.toHaveBeenCalled()
+    expect(mockCreateKnowledgeDocumentUploadSession).not.toHaveBeenCalled()
+  })
+})

apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts

@@ -0,0 +1,61 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { createKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
+import { parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { validateFileType } from '@/lib/uploads/utils/validation'
+import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
+import {
+  requireKnowledgeDocumentUploadAccess,
+  requireKnowledgeDocumentUploadActor,
+  requireKnowledgeDocumentUploadBilling,
+} from '@/app/api/knowledge/[id]/documents/uploads/utils'
+import {
+  createKnowledgeDocumentUploadSession,
+  toV2KnowledgeDocumentUpload,
+} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
+
+interface KnowledgeDocumentUploadsRouteParams {
+  params: Promise<{ id: string }>
+}
+
+export const POST = withRouteHandler(
+  async (request: NextRequest, context: KnowledgeDocumentUploadsRouteParams) => {
+    const actor = await requireKnowledgeDocumentUploadActor()
+    if (actor instanceof NextResponse) return actor
+    const parsed = await parseRequest(createKnowledgeDocumentUploadContract, request, context)
+    if (!parsed.success) return parsed.response
+    const { id: knowledgeBaseId } = parsed.data.params
+    const { workspaceId, name, contentType, size, ...metadata } = parsed.data.body
+    const access = await requireKnowledgeDocumentUploadAccess({
+      knowledgeBaseId,
+      workspaceId,
+      userId: actor.id,
+    })
+    if (access instanceof NextResponse) return access
+    const billing = await requireKnowledgeDocumentUploadBilling({
+      workspaceId,
+      userId: actor.id,
+    })
+    if (billing instanceof NextResponse) return billing
+    const fileTypeError = validateFileType(name, contentType)
+    if (fileTypeError) {
+      return NextResponse.json({ error: fileTypeError.message }, { status: 415 })
+    }
+    try {
+      const upload = await createKnowledgeDocumentUploadSession({
+        workspaceId,
+        userId: actor.id,
+        knowledgeBaseId,
+        fileName: name,
+        contentType,
+        fileSize: size,
+        metadata,
+      })
+      return NextResponse.json({ data: toV2KnowledgeDocumentUpload(upload, null) }, { status: 201 })
+    } catch (error) {
+      const classified = uploadSessionErrorResponse(error)
+      if (classified) return classified
+      throw error
+    }
+  }
+)