fix(knowledge): make a capped listing index the most recent items

Cursor Bugbot, high severity. Both connectors listed ascending by updatedAt
and then took the first N indexable rows, so a cap meant "the oldest N".
The sharper consequence was the second-order one: an already-indexed item
that got edited moved past the cap window, stopped being listed, and — since
listingCapped suppresses deletion reconciliation — left a permanently stale
document in the knowledge base.

Ordering now follows from whether the listing is complete or bounded.
Uncapped stays ascending, which is the safe walk: a row updated mid-sync
moves toward the end and may be emitted twice, and the engine dedupes on
externalId. Capped switches to descending so the cap means "the N most
recently active", which is what maxFiles/maxConversations are for. Its own
risk — a row updated mid-sync slipping behind the cursor — is already
covered, because a capped listing is declared incomplete and the next sync
picks the row up.

The keyset comparison flips with the order (gt/lt) or page two would repeat
page one; both filter builders take the direction explicitly. Verified
against the live database: capping to 3 now returns the three most recently
active conversations rather than the three oldest.

Author: mzxchandra

apps/sim/connectors/sim-conversations/sim-conversations.test.ts

@@ -105,6 +105,23 @@ describe('buildConversationListingFilters', () => {
     ).toContain('ESCAPE')
   })
 
+  /** The keyset direction must match ORDER BY — see the files connector's note. */
+  it('flips the keyset comparison when the listing is descending', () => {
+    const cursor = { updatedAt: new Date('2026-01-01T00:00:00.000Z'), id: 'mem-1' }
+    const ascending = conditionsOf({ workspaceId: 'ws-1', prefix: '', cursor })
+    const descending = conditionsOf({ workspaceId: 'ws-1', prefix: '', cursor, descending: true })
+
+    const branches = (nodes: MockCondition[]) =>
+      nodes
+        .filter((n) => n.type === 'or')
+        .flatMap((n) => (n.conditions as MockCondition[]) ?? [])
+        .flatMap(flattenMockConditions)
+
+    expect(branches(ascending).some((n) => n.type === 'gt')).toBe(true)
+    expect(branches(descending).some((n) => n.type === 'lt')).toBe(true)
+    expect(branches(descending).some((n) => n.type === 'gt')).toBe(false)
+  })
+
   it('adds a keyset clause only when paginating', () => {
     const first = conditionsOf({ workspaceId: 'ws-1', prefix: '' })
     const next = conditionsOf({

apps/sim/connectors/sim-conversations/sim-conversations.ts

@@ -1,6 +1,6 @@
 import { db } from '@sim/db'
 import { memory, memorySecretProvenance } from '@sim/db/schema'
-import { and, asc, eq, gt, isNull, or, type SQL, sql } from 'drizzle-orm'
+import { and, asc, desc, eq, gt, isNull, lt, or, type SQL, sql } from 'drizzle-orm'
 import { readBoundMemorySecretProvenance } from '@/lib/memory/secret-provenance'
 import { simConversationsConnectorMeta } from '@/connectors/sim-conversations/meta'
 import type {
@@ -91,6 +91,8 @@ export function buildConversationListingFilters(args: {
   workspaceId: string
   prefix: string
   cursor?: Cursor
+  /** Must match the query's ORDER BY, or the keyset walks the wrong way. */
+  descending?: boolean
 }): SQL[] {
   const filters: SQL[] = [eq(memory.workspaceId, args.workspaceId), isNull(memory.deletedAt)]
 
@@ -99,10 +101,11 @@ export function buildConversationListingFilters(args: {
   }
 
   if (args.cursor) {
+    const beyond = args.descending ? lt : gt
     filters.push(
       or(
-        gt(memory.updatedAt, args.cursor.updatedAt),
-        and(eq(memory.updatedAt, args.cursor.updatedAt), gt(memory.id, args.cursor.id))
+        beyond(memory.updatedAt, args.cursor.updatedAt),
+        and(eq(memory.updatedAt, args.cursor.updatedAt), beyond(memory.id, args.cursor.id))
       ) as SQL
     )
   }
@@ -235,6 +238,13 @@ export const simConversationsConnector: ConnectorConfig = {
     const minMessages = parseOptionalPositiveInt(sourceConfig.minMessages) ?? 1
     const maxConversations = parseOptionalPositiveInt(sourceConfig.maxConversations) ?? 0
 
+    /**
+     * See the matching note in the files connector: ascending is the safe walk for a
+     * complete listing, but under a cap it would mean "the oldest N" and would freeze
+     * an already-indexed conversation the moment it received a new message.
+     */
+    const descending = maxConversations > 0
+
     const rows = await db
       .select(CONVERSATION_ROW_COLUMNS)
       .from(memory)
@@ -244,10 +254,14 @@ export const simConversationsConnector: ConnectorConfig = {
             workspaceId,
             prefix: readPrefix(sourceConfig),
             cursor: cursor ? decodeCursor(cursor) : undefined,
+            descending,
           })
         )
       )
-      .orderBy(asc(memory.updatedAt), asc(memory.id))
+      .orderBy(
+        descending ? desc(memory.updatedAt) : asc(memory.updatedAt),
+        descending ? desc(memory.id) : asc(memory.id)
+      )
       .limit(PAGE_SIZE)
 
     const items: ExternalDocument[] = []

apps/sim/connectors/sim-files/sim-files.test.ts

@@ -184,6 +184,32 @@ describe('buildFileListingFilters', () => {
     expect(columnNames(nodes.filter((n) => n.type === 'isNull'))).toContain('folderId')
   })
 
+  /**
+   * The keyset direction must match ORDER BY. Ascending walks forward with `gt`;
+   * descending (used when a cap is set, so the cap means "most recently active N"
+   * rather than "oldest N") must walk with `lt` or the second page repeats page one.
+   */
+  it('flips the keyset comparison when the listing is descending', () => {
+    const cursor = { updatedAt: new Date('2026-01-01T00:00:00.000Z'), id: 'file-1' }
+    const ascending = orBranches(
+      conditionsOf({ workspaceId: 'ws-1', folderIds: null, rootOnly: false, cursor })
+    )
+    const descending = orBranches(
+      conditionsOf({
+        workspaceId: 'ws-1',
+        folderIds: null,
+        rootOnly: false,
+        cursor,
+        descending: true,
+      })
+    )
+
+    expect(ascending.some((node) => node.type === 'gt')).toBe(true)
+    expect(ascending.some((node) => node.type === 'lt')).toBe(false)
+    expect(descending.some((node) => node.type === 'lt')).toBe(true)
+    expect(descending.some((node) => node.type === 'gt')).toBe(false)
+  })
+
   it('adds a keyset clause only when paginating', () => {
     const first = conditionsOf({ workspaceId: 'ws-1', folderIds: null, rootOnly: false })
     const next = conditionsOf({

apps/sim/connectors/sim-files/sim-files.ts

@@ -2,7 +2,7 @@ import { db } from '@sim/db'
 import { workspaceFiles } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
-import { and, asc, eq, gt, inArray, isNull, or, type SQL } from 'drizzle-orm'
+import { and, asc, desc, eq, gt, inArray, isNull, lt, or, type SQL } from 'drizzle-orm'
 import { getBaseUrl } from '@/lib/core/utils/urls'
 import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers'
 import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
@@ -143,6 +143,8 @@ export function buildFileListingFilters(args: {
   folderIds: string[] | null
   rootOnly: boolean
   cursor?: Cursor
+  /** Must match the query's ORDER BY, or the keyset walks the wrong way. */
+  descending?: boolean
 }): SQL[] {
   const filters: SQL[] = [
     eq(workspaceFiles.workspaceId, args.workspaceId),
@@ -159,12 +161,13 @@ export function buildFileListingFilters(args: {
   }
 
   if (args.cursor) {
+    const beyond = args.descending ? lt : gt
     filters.push(
       or(
-        gt(workspaceFiles.updatedAt, args.cursor.updatedAt),
+        beyond(workspaceFiles.updatedAt, args.cursor.updatedAt),
         and(
           eq(workspaceFiles.updatedAt, args.cursor.updatedAt),
-          gt(workspaceFiles.id, args.cursor.id)
+          beyond(workspaceFiles.id, args.cursor.id)
         )
       ) as SQL
     )
@@ -302,6 +305,22 @@ export const simFilesConnector: ConnectorConfig = {
       throw error
     }
 
+    /**
+     * Ordering follows from whether this listing is complete or bounded.
+     *
+     * Uncapped, ascending is the safe walk: a row updated mid-sync moves toward the
+     * end and may be emitted twice, which the engine dedupes by `externalId`.
+     *
+     * Capped, ascending would mean "the oldest N" — and worse, an already-indexed
+     * file that is edited moves past the cap window, stops being listed, and (because
+     * `listingCapped` suppresses deletion) leaves a permanently stale document behind.
+     * Descending makes the cap mean "the N most recently active", which is what a
+     * `maxFiles` limit is for. Its own risk — a row updated mid-sync slipping behind
+     * the cursor — is already covered, since a capped listing is declared incomplete
+     * and the next sync picks the row up.
+     */
+    const descending = maxFiles > 0
+
     const rows = await db
       .select(FILE_ROW_COLUMNS)
       .from(workspaceFiles)
@@ -312,10 +331,14 @@ export const simFilesConnector: ConnectorConfig = {
             folderIds: scope.folderIds,
             rootOnly: scope.rootOnly,
             cursor: cursor ? decodeCursor(cursor) : undefined,
+            descending,
           })
         )
       )
-      .orderBy(asc(workspaceFiles.updatedAt), asc(workspaceFiles.id))
+      .orderBy(
+        descending ? desc(workspaceFiles.updatedAt) : asc(workspaceFiles.updatedAt),
+        descending ? desc(workspaceFiles.id) : asc(workspaceFiles.id)
+      )
       .limit(PAGE_SIZE)
 
     const items: ExternalDocument[] = []