fix(knowledge): only block deletion reconciliation on a truncated listing

Cursor Bugbot, medium severity. Both connectors set listingCapped whenever
takeIndexableWithinCap reported capReached — but capReached means the budget
is spent, not that anything was left behind. A source that runs out at
exactly maxFiles/maxConversations produces a COMPLETE listing, and marking it
partial suppressed shouldReconcileDeletions permanently, so an item deleted
at the source could never leave the knowledge base.

Extracts the decision as isListingTruncated in connectors/utils.ts, matching
decideTaskCap in the Asana connector: truncated only when this page dropped
items, or the budget ran out with more pages still available. Both connectors
now share it rather than repeating the cap bookkeeping.

Mutation-checked: collapsing it back to `return args.capReached` reproduces
the reported bug and fails the exhausted-at-exactly-the-cap test.

Author: mzxchandra

apps/sim/connectors/sim-conversations/sim-conversations.ts

@@ -11,6 +11,7 @@ import type {
 } from '@/connectors/types'
 import {
   CONNECTOR_MAX_FILE_BYTES,
+  isListingTruncated,
   isSkippedDocument,
   markSkipped,
   parseTagDate,
@@ -285,8 +286,20 @@ export const simConversationsConnector: ConnectorConfig = {
     syncContext.simConversationsIndexed = indexedSoFar + indexableCount
 
     if (capReached) {
-      // The listing stops short of the source, so it cannot be used to infer deletions.
-      syncContext.listingCapped = true
+      /**
+       * Only a genuinely truncated listing blocks deletion reconciliation — see the
+       * matching note in the files connector. Exhausting the source at exactly
+       * `maxConversations` is a complete listing.
+       */
+      if (
+        isListingTruncated({
+          capReached,
+          droppedFromPage: documents.length < items.length,
+          morePagesAvailable: pageFilled,
+        })
+      ) {
+        syncContext.listingCapped = true
+      }
       return { documents, hasMore: false }
     }
 

apps/sim/connectors/sim-files/sim-files.ts

@@ -23,6 +23,7 @@ import type {
 } from '@/connectors/types'
 import {
   CONNECTOR_MAX_FILE_BYTES,
+  isListingTruncated,
   isSkippedDocument,
   markSkipped,
   parseMultiValue,
@@ -381,8 +382,23 @@ export const simFilesConnector: ConnectorConfig = {
     syncContext.simFilesIndexed = indexedSoFar + indexableCount
 
     if (capReached) {
-      // The listing stops short of the source, so it cannot be used to infer deletions.
-      syncContext.listingCapped = true
+      /**
+       * Hitting the cap is not the same as truncating. If the source ran out at
+       * exactly `maxFiles` — nothing dropped from this page and no further page —
+       * the listing is complete and safe to reconcile deletions against. Marking it
+       * capped anyway would suppress reconciliation forever, so a file deleted at the
+       * source would never leave the knowledge base. Mirrors `decideTaskCap`'s
+       * `droppedFromPage || (hitLimit && morePagesAvailable)` in the Asana connector.
+       */
+      if (
+        isListingTruncated({
+          capReached,
+          droppedFromPage: documents.length < items.length,
+          morePagesAvailable: pageFilled,
+        })
+      ) {
+        syncContext.listingCapped = true
+      }
       return { documents, hasMore: false }
     }
 

apps/sim/connectors/utils-listing-cap.test.ts

@@ -0,0 +1,37 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { isListingTruncated } from '@/connectors/utils'
+
+/**
+ * `capReached` means the budget is spent, not that anything was left behind.
+ * Conflating the two suppresses deletion reconciliation on a complete listing, so a
+ * source deletion could never propagate to the knowledge base.
+ */
+describe('isListingTruncated', () => {
+  it('is not truncated when the source runs out exactly at the cap', () => {
+    expect(
+      isListingTruncated({ capReached: true, droppedFromPage: false, morePagesAvailable: false })
+    ).toBe(false)
+  })
+
+  it('is truncated when this page had to drop items', () => {
+    expect(
+      isListingTruncated({ capReached: true, droppedFromPage: true, morePagesAvailable: false })
+    ).toBe(true)
+  })
+
+  it('is truncated when the budget ran out with more pages left', () => {
+    expect(
+      isListingTruncated({ capReached: true, droppedFromPage: false, morePagesAvailable: true })
+    ).toBe(true)
+  })
+
+  /** A full page with budget to spare is just pagination, not truncation. */
+  it('is not truncated when more pages remain but the cap was not reached', () => {
+    expect(
+      isListingTruncated({ capReached: false, droppedFromPage: false, morePagesAvailable: true })
+    ).toBe(false)
+  })
+})

apps/sim/connectors/utils.ts

@@ -229,6 +229,25 @@ export function takeIndexableWithinCap<T>(
   return { documents, indexableCount, capReached: alreadyIndexed + indexableCount >= max }
 }
 
+/**
+ * Whether a capped listing actually stopped short of the source.
+ *
+ * `takeIndexableWithinCap` reports `capReached` — the budget is spent — which is NOT
+ * the same as truncation. A source that runs out at exactly the cap yields a complete
+ * listing, and marking it truncated would suppress deletion reconciliation forever,
+ * so an item deleted at the source could never leave the knowledge base.
+ *
+ * Same rule as `decideTaskCap` in the Asana connector: truncated only when this page
+ * dropped items, or the budget ran out with more pages still available.
+ */
+export function isListingTruncated(args: {
+  capReached: boolean
+  droppedFromPage: boolean
+  morePagesAvailable: boolean
+}): boolean {
+  return args.droppedFromPage || (args.capReached && args.morePagesAvailable)
+}
+
 /**
  * Raised by a connector when a file exceeds its size cap mid-download — i.e. the
  * listing did not report a size, so the limit is only discovered while streaming.