improvement(uploads): persist multipart sessions in postgres

Author: TheodoreSpeaks

apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx

@@ -78,7 +78,6 @@ cat > /tmp/cors.json <<'EOF'
       "AllowedOrigins": ["https://sim.yourdomain.com"],
       "AllowedMethods": ["GET", "PUT"],
       "AllowedHeaders": ["*"],
-      "ExposeHeaders": ["ETag"],
       "MaxAgeSeconds": 3600
     }
   ]
@@ -91,10 +90,6 @@ for name in workspace-files knowledge-base execution-files chat-files \
 done
 ```
 
-<Callout type="warn">
-  `ExposeHeaders` **must** include `ETag`. Files larger than 50 MB use multipart uploads, and the browser reads each part's `ETag` to complete the upload — CORS hides the header otherwise and large uploads fail at the final step.
-</Callout>
-
 Set `AllowedOrigins` to your exact Sim origin (scheme + host, no trailing slash). Add every origin users reach Sim from, including an apex/`www` pair if both are live.
 
 </Step>
@@ -239,15 +234,15 @@ AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME=workspace-logos
 
 Direct browser uploads require a Blob service CORS rule on the storage account. Allow your exact
 Sim origin, `GET` and `PUT`, the `Content-Type` header, and the `x-ms-*` prefix used by signed blob
-and metadata headers:
+and metadata headers. Small-file uploads also send `If-None-Match` so a signed URL cannot overwrite
+an existing final object:
 
 ```bash
 az storage cors add \
   --services b \
   --methods GET PUT \
   --origins https://sim.yourdomain.com \
-  --allowed-headers content-type 'x-ms-*' \
-  --exposed-headers ETag \
+  --allowed-headers content-type if-none-match 'x-ms-*' \
   --max-age 3600 \
   --account-name mystorageaccount \
   --account-key '<account-key>'
@@ -295,7 +290,7 @@ cat > /tmp/cors.json <<'EOF'
     "method": ["GET", "PUT"],
     "responseHeader": [
       "Content-Type",
-      "ETag",
+      "x-goog-if-generation-match",
       "x-goog-meta-uploadid",
       "x-goog-meta-originalname",
       "x-goog-meta-uploadedat",
@@ -319,7 +314,7 @@ done
 ```
 
 <Callout type="info">
-  Header names must be listed individually — GCS CORS matches `responseHeader` entries exactly and does not support wildcards like `x-goog-meta-*`. `ETag` is required because large-file multipart uploads read each part's `ETag` from the browser, and CORS hides the header otherwise.
+  Header names must be listed individually — GCS CORS matches `responseHeader` entries exactly and does not support wildcards like `x-goog-meta-*`. `x-goog-if-generation-match` makes small-file uploads create-only; Sim obtains multipart ETags from GCS during completion rather than exposing them to the browser.
 </Callout>
 
 </Step>
@@ -467,25 +462,25 @@ The same browser-reachability and CORS requirements apply.
   </Tab>
 </Tabs>
 
-## Configure temporary upload cleanup
+## Configure incomplete multipart cleanup
 
-Sim stages every direct upload under the `upload-sessions/` prefix before promoting it to its final,
-immutable object key. Apply the cleanup policy to **every** purpose-specific bucket or container
-configured above:
+Sim uploads directly to a create-only final object key and keeps upload-session state in PostgreSQL.
+The cleanup cron claims expired sessions before deleting an uploaded object or aborting its provider
+multipart state. Configure provider lifecycle cleanup as a second line of defense for multipart
+state that outlives its database row:
 
-- On AWS S3 and Google Cloud Storage, expire objects under `upload-sessions/` after two days and
-  abort incomplete multipart uploads after two days.
-- On Azure Blob, expire committed blobs under `upload-sessions/` after two days. Azure automatically
-  removes uncommitted blocks after seven days.
-- For an S3-compatible provider, configure both rules when its lifecycle implementation supports
-  them. Check the provider's documentation because lifecycle feature support varies.
+- On AWS S3 and Google Cloud Storage, abort incomplete multipart uploads after two days on every
+  purpose-specific bucket.
+- Azure automatically removes uncommitted blocks after seven days.
+- For an S3-compatible provider, configure incomplete-multipart cleanup when its lifecycle
+  implementation supports it. Check the provider's documentation because support varies.
 
-The two-day window exceeds the 24-hour upload-token lifetime and leaves time to retry completion.
-Do not apply this prefix rule to final objects outside `upload-sessions/`.
+The provider window should exceed the 24-hour upload-session lifetime so an in-progress completion
+can still recover. Do not add an object-expiration rule for final upload keys.
 
 <Callout type="warning">
-  Configure both expiration and incomplete-multipart cleanup where available. Expiring staged
-  objects alone does not necessarily remove abandoned multipart parts.
+  Object expiration and incomplete-multipart cleanup are different lifecycle operations. Configure
+  the incomplete-multipart operation; expiring objects does not remove abandoned multipart parts.
 </Callout>
 
 ## Verify it works

apps/docs/content/docs/en/platform/self-hosting/troubleshooting.mdx

@@ -193,7 +193,7 @@ Both pods must have `REDIS_URL`. On Helm they share one Secret, so setting it un
 
 The bucket's CORS policy does not allow your Sim origin. Uploads go directly from the browser to object storage via presigned `PUT`, so server-side configuration being correct is not enough.
 
-If small uploads succeed but files over 50 MB fail at the last step, `ETag` is missing from the CORS exposed headers — multipart uploads read it from the browser. See [Object Storage](/platform/self-hosting/object-storage).
+If small uploads succeed but files over 50 MB fail during completion, check the app logs for the provider's part-listing request. The server completes multipart uploads from provider-authoritative state; for S3, its identity needs `s3:ListMultipartUploadParts`. See [Object Storage](/platform/self-hosting/object-storage).
 
 ## Agent Output Arrives All at Once
 

apps/docs/content/docs/en/platform/self-hosting/verify.mdx

@@ -18,7 +18,7 @@ Run this after a first install, after an upgrade, and after a restore. Each step
 | 4 | Open the same workflow in a second browser window and edit | Cross-replica collaboration | With >1 replica this needs [Redis](/platform/self-hosting/redis) |
 | 5 | Paste a model API key in settings and run a two-block workflow | Execution engine, credential encryption, outbound network | App logs; check `ENCRYPTION_KEY` is set and outbound egress is allowed |
 | 6 | Upload a small file in Files | File storage end to end | With object storage configured: presigned URL + bucket CORS. On local disk: the upload proxies through the app |
-| 7 | Upload a file larger than 50 MB | Multipart upload path (object storage only) | Confirm `ETag` is in the bucket's CORS exposed headers |
+| 7 | Upload a file larger than 50 MB | Multipart upload path (object storage only) | Check app logs for provider part-listing or completion errors |
 | 8 | Create a knowledge base and upload a PDF | Document parsing, embeddings, pgvector | Needs a hosted embedding provider — see below |
 | 9 | Invite a teammate from workspace settings | Email delivery | App logs for the mailer; see [Email](/platform/self-hosting/email) |
 | 10 | Connect an integration account | OAuth configuration | Redirect URI mismatch → see [Integrations & OAuth](/platform/self-hosting/integrations-oauth) |
@@ -72,7 +72,7 @@ All six should be present on Compose: `simstudio`, `realtime`, `db`, `redis`, `c
 
 **Step 5 fails — execution errors.** Check outbound connectivity to the model provider, then the app logs. If the error is about decrypting a credential, `ENCRYPTION_KEY` differs from the one that encrypted it.
 
-**Step 6 or 7 fails.** With object storage configured, a CORS error in the browser console means the bucket policy does not allow your Sim origin; step 7 failing while step 6 passes specifically means `ETag` is missing from the exposed headers. On local-disk storage there is no CORS involved — uploads proxy through the app, so look at the app logs and the proxy body-size limit instead.
+**Step 6 or 7 fails.** With object storage configured, a CORS error in the browser console means the bucket policy does not allow your Sim origin or the signed upload headers. If step 7 fails only during completion, check the app logs and verify the server identity can list multipart parts (for S3, `s3:ListMultipartUploadParts`). On local-disk storage there is no CORS involved — uploads proxy through the app, so look at the app logs and the proxy body-size limit instead.
 
 **Step 8 fails — knowledge base upload errors.** Knowledge bases need a hosted embedding provider — OpenAI, Azure OpenAI, or Gemini. There is no local embedding backend. If a key is set, check pgvector is installed on the database.
 

apps/docs/openapi-v2-files-audit.json

@@ -435,14 +435,6 @@
             "$ref": "#/components/parameters/UploadTokenHeader"
           }
         ],
-        "requestBody": {
-          "required": true,
-          "content": {
-            "application/json": {
-              "schema": {}
-            }
-          }
-        },
         "responses": {
           "200": {
             "description": "The completed upload and registered file.",

apps/docs/openapi-v2-knowledge.json

@@ -1008,16 +1008,6 @@
             "$ref": "#/components/parameters/WorkspaceIdQuery"
           }
         ],
-        "requestBody": {
-          "required": true,
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/CompleteUploadBody"
-              }
-            }
-          }
-        },
         "responses": {
           "200": {
             "description": "The completed upload and queued knowledge document.",
@@ -2197,41 +2187,6 @@
           }
         }
       },
-      "CompleteUploadBody": {
-        "oneOf": [
-          {
-            "type": "object",
-            "additionalProperties": false,
-            "required": ["parts"],
-            "properties": {
-              "parts": {
-                "type": "array",
-                "minItems": 1,
-                "maxItems": 640,
-                "items": {
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": ["partNumber"],
-                  "properties": {
-                    "partNumber": {
-                      "type": "integer",
-                      "minimum": 1
-                    },
-                    "etag": {
-                      "type": "string",
-                      "minLength": 1
-                    }
-                  }
-                }
-              }
-            }
-          },
-          {
-            "type": "object",
-            "additionalProperties": false
-          }
-        ]
-      },
       "DocumentSummary": {
         "type": "object",
         "description": "Summary representation of a document, returned in list operations and as the upload acknowledgement.",

apps/docs/openapi-v2-tables.json

@@ -3468,14 +3468,6 @@
             "$ref": "#/components/parameters/UploadTokenHeader"
           }
         ],
-        "requestBody": {
-          "required": true,
-          "content": {
-            "application/json": {
-              "schema": {}
-            }
-          }
-        },
         "responses": {
           "200": {
             "description": "The queued import resource.",

apps/sim/app/api/cron/cleanup-tasks/route.ts

@@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { verifyCronAuth } from '@/lib/auth/internal'
 import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { cleanupExpiredUploadSessions } from '@/lib/uploads/upload-session/service'
 
 export const dynamic = 'force-dynamic'
 
@@ -13,11 +14,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     const authError = verifyCronAuth(request, 'task cleanup')
     if (authError) return authError
 
+    const uploadSessions = await cleanupExpiredUploadSessions()
     const result = await dispatchCleanupJobs('cleanup-tasks')
 
-    logger.info('Task cleanup jobs dispatched', result)
+    logger.info('Task cleanup jobs dispatched', { ...result, uploadSessions })
 
-    return NextResponse.json({ triggered: true, ...result })
+    return NextResponse.json({ triggered: true, ...result, uploadSessions })
   } catch (error) {
     logger.error('Failed to dispatch task cleanup jobs:', { error })
     return NextResponse.json({ error: 'Failed to dispatch task cleanup' }, { status: 500 })

apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts

@@ -22,15 +22,14 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
   if (!parsed.success) return parsed.response
 
   try {
-    const session = getOwnedUploadSession({
+    const session = await getOwnedUploadSession({
       uploadId: parsed.data.params.uploadId,
       uploadToken: parsed.data.headers['upload-token'],
       userId: actor.id,
     })
     await reauthorizeUploadPurpose(actor.id, session)
     const completed = await completeUploadSession({
       session,
-      completion: parsed.data.body,
       finalize: (claimed) => finalizeUploadPurpose({ session: claimed, actor, request }),
     })
     return NextResponse.json({

apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts

@@ -17,7 +17,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
   if (!parsed.success) return parsed.response
 
   try {
-    const session = getOwnedUploadSession({
+    const session = await getOwnedUploadSession({
       uploadId: parsed.data.params.uploadId,
       uploadToken: parsed.data.headers['upload-token'],
       userId: actor.id,

apps/sim/app/api/files/uploads/[uploadId]/route.ts

@@ -21,7 +21,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Upl
   if (!parsed.success) return parsed.response
 
   try {
-    const session = getOwnedUploadSession({
+    const session = await getOwnedUploadSession({
       uploadId: parsed.data.params.uploadId,
       uploadToken: parsed.data.headers['upload-token'],
       userId: actor.id,