From 129630ec9b86accd0f133a826156bed4e16dcc81 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Tue, 11 Aug 2026 14:43:21 +0200 Subject: [PATCH 01/16] feat: implement destructive reconciliation --- .../paginators/MessageIntervalPaginator.ts | 208 +++++++- src/pagination/paginators/MessagePaginator.ts | 1 + src/thread.ts | 20 +- .../paginators/MessagePaginator.test.ts | 471 ++++++++++++++++++ 4 files changed, 688 insertions(+), 12 deletions(-) diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index 75348e423..c74a8049e 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -142,6 +142,37 @@ export type MessagePaginatorOptions = { paginatorOptions?: PaginatorOptions; }; +/** + * Options for {@link MessageIntervalPaginator.mergeNewestPage} that enable destructive + * reconciliation — removing messages that were hard-deleted (by anyone) while the client was + * offline. A hard delete emits no event to other clients, and the merge is otherwise additive, so + * without this such a message lingers as a ghost after reconnect. + * + * With NO options, `mergeNewestPage` still prunes any loaded message that falls WITHIN the returned + * page's `created_at` span but is absent from it — unconditionally safe (a message that arrived live + * during the caller's fetch is always strictly newer than the newest returned message, so it can + * never fall in that span). The options widen the reconcilable window: + */ +export type MergeNewestPageOptions = { + /** + * The `limit` the caller passed to the query that produced the page. Lets reconciliation tell + * "the page reached the channel's oldest message" (a returned count short of the request) from + * "the page is full and older messages remain". Only then may it prune loaded messages OLDER than + * the oldest returned message (e.g. the oldest loaded message was the one deleted). Clamped to the + * server's max page size so an over-request cannot be mistaken for reaching the start. + */ + requestedLimit?: number; + /** + * A snapshot of the loaded message ids taken BEFORE the caller's fetch await. Required to prune + * messages NEWER than the newest returned message (a hard-deleted newest message) and to reconcile + * an empty page (a fully-emptied channel): at/above that top edge a just-deleted message and a + * message that arrived live during the fetch are indistinguishable by timestamp — only the + * pre-fetch snapshot separates them (a live arrival is not in it). Must be captured before the + * await; the paginator's own items at merge time already include any live arrival. + */ + candidateIds?: ReadonlySet; +}; + /** * MessageIntervalPaginator allows configuring backend request sort, while keeping internal item ordering stable. * Filtering of ingested items is still limited to local predicates (`filterQueryResults`). @@ -549,23 +580,30 @@ export class MessageIntervalPaginator extends BasePaginator< * page is new). Merge in place: existing items are reconciled by id (edits, soft deletes), new * items are appended and every already loaded item (including older pages already paged in) is * kept. `hasMoreTail`/`cursor.tailward` are left as-is so the page can be any size, so deriving - * "has older items" from its length would wrongly clear it while older items remain. + * "has older items" from its length would wrongly clear it while older items remain. Then + * destructive reconciliation ({@link reconcileLoadedAgainstPage}) removes any loaded message + * that the authoritative page proves was hard-deleted while offline (see that method + the + * {@link MergeNewestPageOptions} for the exact, safe window). * * 2. DISJOINT - the incoming page shares no id with the loaded head (at least a full page is new). * Merging would weld the two across the gap (the interval merge treats two head intervals as * overlapping when one reaches further headward), hiding the items in between with no way to * reach them. Instead the loaded set is discarded and rebuilt from the incoming page as a fresh * contiguous head (`hasMoreTail: true`, cursor reanchored to the page's oldest item) so the - * gap and older history load again when paginating older. + * gap and older history load again when paginating older. No separate reconciliation is needed: + * the rebuilt window IS the server truth, so any hard-deleted message is simply absent from it. + * + * Never blanks the loaded set. Noop unless the newest slice is both loaded AND the interval + * currently in view (the head interval is anchored at the head and active); when the caller has + * jumped to a separate older window the merge is skipped so their position is preserved, and the + * incoming page is picked up on a later load. An empty page never wipes the list on its own, but + * — given a pre-fetch snapshot via `options.candidateIds` — is reconciled as "the channel has no + * messages" (every server-confirmed loaded message removed). * - * Both paths emit exactly once and never blank the loaded set. Noop unless the page is non empty - * and the newest slice is both loaded AND the interval currently in view (the head interval is - * anchored at the head and active); when the caller has jumped to a separate older window the - * merge is skipped so their position is preserved, and the incoming page is picked up on a later - * load. + * @param page - The fetched newest window (may be empty). `created_at` order is normalized on ingest. + * @param options - See {@link MergeNewestPageOptions}. Omit to prune only within the page's own span. */ - mergeNewestPage = (page: LocalMessage[]) => { - if (!page?.length) return; + mergeNewestPage = (page: LocalMessage[], options?: MergeNewestPageOptions) => { const headInterval = this.itemIntervals[0] as Interval | undefined; if (!headInterval?.isHead) return; // Only reconcile when the head is the interval currently in view. If the caller jumped to a @@ -574,6 +612,14 @@ export class MessageIntervalPaginator extends BasePaginator< // their position (the newest page is picked up on scroll / a later load). if (!this.isActiveInterval(headInterval)) return; + if (!page?.length) { + // Empty page: never blanks the list by itself. Only when the caller supplied a pre-fetch + // snapshot do we treat it as authoritative "channel emptied" and remove ghosts (a message that + // arrived live during the fetch is excluded by the snapshot). + this.reconcileLoadedAgainstPage([], options); + return; + } + const loadedIds = new Set(headInterval.itemIds); const overlapsLoadedHead = page.some((item) => loadedIds.has(this.getItemId(item))); @@ -614,8 +660,152 @@ export class MessageIntervalPaginator extends BasePaginator< // nothing newer to load. hasMoreTail / cursor are deliberately preserved (see above). hasMoreHead: false, }); + + // With the newest page merged in, drop any loaded message the page proves was hard-deleted. + this.reconcileLoadedAgainstPage(page, options); }; + /** + * Whether a loaded message is server-confirmed and therefore eligible to be reconciled away when + * absent from an authoritative page. Excludes local-only messages the server has never + * acknowledged — optimistic (`sending`) and `failed` sends, and client-side `error` placeholders — + * so a legitimately-unsent message is never mistaken for a hard delete. + */ + protected isServerConfirmedMessage(message: LocalMessage): boolean { + return ( + message.status !== 'sending' && + message.status !== 'failed' && + message.type !== 'error' + ); + } + + /** + * Destructive half of {@link mergeNewestPage}: remove loaded messages that the freshly-fetched + * newest `page` proves were hard-deleted while offline (a hard delete emits no event to other + * clients, and the merge is additive, so they would otherwise linger forever). + * + * The reconcilable window is derived ENTIRELY from what the page returned — never a hardcoded page + * size — so it can only ever remove messages the page actually covers: + * + * - WITHIN the page's span (`oldest returned < created_at < newest returned`): a server-confirmed + * loaded message absent from the page was hard-deleted. Safe with no snapshot — a message that + * arrived live during the caller's fetch is always strictly newer than the newest returned + * message, so it can never fall in this span. + * - BELOW the oldest returned message: only reconcilable when the page reached the channel's oldest + * message (`requestedLimit` given and the page came back short, clamped to the server max page + * size). Otherwise older messages simply were not fetched and are left untouched. + * - AT/ABOVE the newest returned message (a hard-deleted newest message) and the empty-page case: + * only reconcilable with a pre-fetch `candidateIds` snapshot, which alone distinguishes a ghost + * from a live arrival at that top edge. + * + * Removal goes through {@link removeItem} (batched) so the item index, intervals, shared message + * store and — via the {@link MessagePaginator} override — the tracked last message all stay + * correct, then the active window is re-emitted once. + */ + protected reconcileLoadedAgainstPage( + page: LocalMessage[], + options?: MergeNewestPageOptions, + ) { + const headInterval = this.itemIntervals[0] as Interval | undefined; + if (!headInterval?.isHead || !this.isActiveInterval(headInterval)) return; + + const loadedIds = headInterval.itemIds; + const candidateIds = options?.candidateIds; + + // Empty page → the channel has no messages. Every server-confirmed loaded message is gone, but + // only remove ids from the pre-fetch snapshot so a message that landed during the fetch survives. + if (!page.length) { + if (!candidateIds) return; + const toRemove = loadedIds.filter((id) => { + if (!candidateIds.has(id)) return false; + const message = this.getItem(id); + return !!message && this.isServerConfirmedMessage(message); + }); + this.removeReconciledIds(toRemove); + return; + } + + const pageIds = new Set(page.map((message) => this.getItemId(message))); + const newestReturnedTs = getMessageCreatedAtTimestamp(page[page.length - 1]); + const oldestReturnedTs = getMessageCreatedAtTimestamp(page[0]); + + // Only extend below the oldest returned message when the page proves it reached the channel's + // oldest message: it came back shorter than requested. Clamp the request to the server's max page + // size so asking for MORE than one page can return (an over-request) is not mistaken for reaching + // the start — in that case the shortfall is the server capping, not the channel ending. + const { requestedLimit } = options ?? {}; + const reachedChannelStart = + typeof requestedLimit === 'number' && + page.length < Math.min(requestedLimit, DEFAULT_CHANNEL_MESSAGE_LIST_PAGE_SIZE); + const windowLowTs = reachedChannelStart + ? Number.NEGATIVE_INFINITY + : (oldestReturnedTs ?? Number.POSITIVE_INFINITY); + + const toRemove: string[] = []; + for (const id of loadedIds) { + if (pageIds.has(id)) continue; // present on the server → keep + const message = this.getItem(id); + if (!message || !this.isServerConfirmedMessage(message)) continue; // local-only → keep + const ts = getMessageCreatedAtTimestamp(message); + if (ts === null) continue; // no server timestamp (optimistic) → keep + + if (newestReturnedTs !== null && ts >= newestReturnedTs) { + // At/above the newest returned message: a hard-deleted newest message and a message that + // arrived live during the fetch are indistinguishable here. Only remove ids present in the + // pre-fetch snapshot, which excludes live arrivals. (The newest returned message itself is + // in the page, so it is already skipped above.) + if (candidateIds?.has(id)) toRemove.push(id); + continue; + } + // Strictly within the page's span (below the newest returned message): a live arrival can never + // be here, so the absence is a hard delete regardless of a snapshot. + if (ts > windowLowTs) toRemove.push(id); + } + + this.removeReconciledIds(toRemove); + } + + /** + * Remove a set of reconciled (hard-deleted) ids in one batch — coalescing the shared-store fan-out + * to a single flush — then flush the deferred window publish so the list drops the ghosts + * synchronously (blanking to `[]` if the active window emptied, per {@link flushWindowPublish}). + * Finally, mirror the removal into the offline DB so a cold start does not re-seed the ghosts from + * SQLite. No-op for an empty set, so an unaffected merge does not touch state a second time. + */ + private removeReconciledIds(ids: string[]) { + if (!ids.length) return; + this._itemIndex.batch(() => { + for (const id of ids) this.removeItem({ id }); + }); + this.flushPendingPublishes(); + this.purgeReconciledFromOfflineDb(ids); + } + + /** + * Mirror a destructive reconciliation into the offline DB. A hard delete performed while the client + * was offline reaches it via no event, so the offline store never ran its own hard-delete for these + * ids; the reconnect query re-hydrates the DB by UPSERT (which never removes what is absent from the + * page), so without this the ghosts survive in SQLite and a cold start would re-seed them after the + * in-memory list already dropped them. + * + * Owned by the state layer, NOT the SDK: the SDK only supplies the platform `offlineDb` + * implementation and never orchestrates DB writes for reconciliation. Fire-and-forget and + * best-effort — the in-memory removal is the source of truth for the live list, so a failed/absent + * DB write is no worse than before (the ghost merely re-appears on a cold start until the next + * reconcile). No-op when offline support is off (`client.offlineDb` undefined). The ids are already + * server-confirmed (the reconcile excludes pending/failed/optimistic), so a plain hard delete — + * without the pending-task teardown of a local failed message — is correct. + */ + private purgeReconciledFromOfflineDb(ids: string[]) { + const offlineDb = this.channel.getClient?.()?.offlineDb; + if (!offlineDb) return; + Promise.all(ids.map((id) => offlineDb.hardDeleteMessage({ id, execute: false }))) + .then((queryBatches) => offlineDb.executeSqlBatch(queryBatches.flat())) + .catch(() => { + // best-effort persistence cleanup — see doc comment; the live list is already correct. + }); + } + protected resolveUnreadBoundaryIdsByTimestamp = ({ lastReadAt, messages, diff --git a/src/pagination/paginators/MessagePaginator.ts b/src/pagination/paginators/MessagePaginator.ts index 51519b4d1..853a40391 100644 --- a/src/pagination/paginators/MessagePaginator.ts +++ b/src/pagination/paginators/MessagePaginator.ts @@ -15,6 +15,7 @@ import { StateStore } from '../../store'; export type { JumpToMessageOptions, + MergeNewestPageOptions, MessageFocusReason, MessageFocusSignal, MessageFocusSignalState, diff --git a/src/thread.ts b/src/thread.ts index 6039a5bfb..156397905 100644 --- a/src/thread.ts +++ b/src/thread.ts @@ -32,6 +32,7 @@ import { MessageComposer } from './messageComposer'; import { MessageOperations } from './messageOperations'; import { WithSubscriptions } from './utils/WithSubscriptions'; import { MessagePaginator } from './pagination'; +import type { MergeNewestPageOptions } from './pagination'; import type { PipelineEvent } from './EventHandlerPipeline'; export type ThreadState = { @@ -325,17 +326,29 @@ export class Thread extends WithSubscriptions { try { const loadedReplyCount = this.messagePaginator.items?.length ?? 0; + const requestedReplyLimit = loadedReplyCount || this.messagePaginator.pageSize; + const reconcileCandidateIds = new Set( + (this.messagePaginator.items ?? []).map((reply) => reply.id), + ); const thread = await this.client.getThreadAndHydrate(this.id, { watch: true, - reply_limit: loadedReplyCount || this.messagePaginator.pageSize, + reply_limit: requestedReplyLimit, + }); + this.hydrateState(thread, { + reconcile: { + requestedLimit: requestedReplyLimit, + candidateIds: reconcileCandidateIds, + }, }); - this.hydrateState(thread); } finally { this.state.partialNext({ isLoading: false }); } }; - public hydrateState = (thread: Thread) => { + public hydrateState = ( + thread: Thread, + options?: { reconcile?: MergeNewestPageOptions }, + ) => { if (thread === this) { // skip if the instances are the same return; @@ -383,6 +396,7 @@ export class Thread extends WithSubscriptions { this.messagePaginator.mergeNewestPage( thread.messagePaginator.state.getLatestValue().items ?? [], + options?.reconcile, ); pendingReplies.forEach((reply) => this.messagePaginator.ingestItem(reply)); // Carry the re-queried thread's last-activity floor so lastMessageAt stays fresh even when the diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index 865df238a..bd23ae5a6 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -1913,6 +1913,477 @@ describe('MessagePaginator', () => { }); }); + describe('mergeNewestPage() — destructive reconciliation', () => { + // A channel (main list) message with a distinct created_at derived from `minute`, so ordering and + // the reconciliation window are unambiguous. Server-confirmed ('received') unless overridden. + const msg = (id: string, minute: number, overrides: Partial = {}) => + createMessage({ + cid: 'channel-id', + id, + created_at: new Date(Date.UTC(2020, 0, 1, 0, minute, 0)).toISOString(), + ...overrides, + }); + + // Loads a newest (head-anchored, active) window from `messages` (any order; sorted on ingest). + const loadHead = ( + messages: LocalMessage[], + { isTail = false }: { isTail?: boolean } = {}, + ) => { + const paginator = new MessagePaginator({ + channel, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + }); + paginator.ingestPage({ page: messages, isHead: true, isTail, setActive: true }); + return paginator; + }; + + const ids = (paginator: MessagePaginator) => + paginator.items?.map((message) => message.id); + + // ── WITHIN the returned page's span (default, no options — unconditionally safe) ────────────── + + it('default (no options): drops a hard-deleted message within the returned page span', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + // m3 hard-deleted while offline: the authoritative newest page comes back without it. + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m4', 4), msg('m5', 5)]); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm4', 'm5']); + expect(paginator.getItem('m3')).toBeUndefined(); + }); + + it('drops several hard-deleted messages in one pass', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + msg('m6', 6), + ]); + // m2 and m4 hard-deleted. + paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3), msg('m5', 5), msg('m6', 6)]); + expect(ids(paginator)).toEqual(['m1', 'm3', 'm5', 'm6']); + }); + + it('reconciles deletions AND additions delivered by the same page', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + ]); + // m3 hard-deleted; m5 and m6 arrived while offline — all in the one authoritative page. + paginator.mergeNewestPage([ + msg('m1', 1), + msg('m2', 2), + msg('m4', 4), + msg('m5', 5), + msg('m6', 6), + ]); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm4', 'm5', 'm6']); + expect(paginator.getItem('m3')).toBeUndefined(); + }); + + it('keeps a soft-deleted message (the server still returns it, so it is in the page)', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + paginator.mergeNewestPage([ + msg('m1', 1), + msg('m2', 2, { type: 'deleted', deleted_at: '2020-01-01T00:10:00.000Z' }), + msg('m3', 3), + ]); + expect(paginator.getItem('m2')).toBeDefined(); + expect(paginator.getItem('m2')?.type).toBe('deleted'); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + }); + + // ── OLDER than the returned page (must be left untouched unless the page reached the start) ─── + + it('leaves loaded messages older than the returned page untouched (full page, start not reached)', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + msg('m6', 6), + ]); + // A full page (returned === requested) covering only the newest four; m4 was hard-deleted, so + // m2 slid into the window: page = [m2,m3,m5,m6]. Older m1 is below the page and MUST stay. + paginator.mergeNewestPage( + [msg('m2', 2), msg('m3', 3), msg('m5', 5), msg('m6', 6)], + { + requestedLimit: 4, + }, + ); + expect(paginator.getItem('m4')).toBeUndefined(); // within-window delete removed + expect(paginator.getItem('m1')).toBeDefined(); // older-than-page kept + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm5', 'm6']); + }); + + it('removes the OLDEST loaded message once the page proves it reached the channel start', () => { + // Whole channel loaded. m1 (oldest) hard-deleted → a request for four returns only three. + const paginator = loadHead( + [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + { + isTail: true, + }, + ); + paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], { + requestedLimit: 4, + }); + expect(paginator.getItem('m1')).toBeUndefined(); + expect(ids(paginator)).toEqual(['m2', 'm3', 'm4']); + }); + + it('keeps the oldest loaded message when the page did NOT prove it reached the start', () => { + const paginator = loadHead( + [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + { + isTail: true, + }, + ); + // A FULL page (returned === requested) that simply does not reach m1 → cannot claim m1 deleted. + paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], { + requestedLimit: 3, + }); + expect(paginator.getItem('m1')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4']); + }); + + // ── AT/ABOVE the newest returned message: trailing deletes (need a pre-fetch snapshot) ──────── + + it('keeps a hard-deleted NEWEST message without a snapshot (documented limitation)', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + // m5 (newest) deleted; the page's newest is now m4. No snapshot ⇒ the top edge is ambiguous. + paginator.mergeNewestPage( + [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + { + requestedLimit: 5, + }, + ); + expect(paginator.getItem('m5')).toBeDefined(); + }); + + it('drops a hard-deleted NEWEST message WITH a snapshot and recomputes lastMessage', () => { + const loaded = [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]; + const paginator = loadHead(loaded); + expect(paginator.lastMessage?.id).toBe('m5'); + const candidateIds = new Set(loaded.map((message) => message.id)); + + paginator.mergeNewestPage( + [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + { + candidateIds, + requestedLimit: 5, + }, + ); + + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4']); + expect(paginator.getItem('m5')).toBeUndefined(); + expect(paginator.lastMessage?.id).toBe('m4'); // tracked latest fell back to the newest survivor + }); + + it('drops multiple hard-deleted trailing messages with a snapshot', () => { + const loaded = [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]; + const paginator = loadHead(loaded); + const candidateIds = new Set(loaded.map((message) => message.id)); + + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { + candidateIds, + requestedLimit: 5, + }); + + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + expect(paginator.lastMessage?.id).toBe('m3'); + }); + + // ── The live-race: a message that arrived during the fetch must never be pruned ────────────── + + it('keeps a message that arrived live during the fetch while dropping a trailing ghost', () => { + // Loaded before the fetch: m1..m4 plus a soon-to-be-deleted newest ghost m5. + const loadedBefore = [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]; + const paginator = loadHead(loadedBefore); + const candidateIds = new Set(loadedBefore.map((message) => message.id)); // snapshot BEFORE fetch + + // During the fetch a brand-new message m6 arrives via WS and is ingested into the head. + paginator.ingestItem(msg('m6', 6)); + expect(paginator.getItem('m6')).toBeDefined(); + + // The server's authoritative page (computed before m6 existed) has m5 deleted and lacks m6. + paginator.mergeNewestPage( + [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + { + candidateIds, + requestedLimit: 5, + }, + ); + + // m5 (ghost, in the snapshot) removed; m6 (live arrival, NOT in the snapshot) kept. + expect(paginator.getItem('m5')).toBeUndefined(); + expect(paginator.getItem('m6')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4', 'm6']); + }); + + // ── Provenance: never reconcile away local-only (unsent) messages ──────────────────────────── + + it('never removes optimistic (sending), failed, or error-type local messages', () => { + const loaded = [ + msg('m1', 1), + msg('sending', 2, { status: 'sending' }), + msg('failed', 3, { status: 'failed' }), + msg('err', 4, { type: 'error' }), + msg('m5', 5), + ]; + const paginator = loadHead(loaded); + const candidateIds = new Set(loaded.map((message) => message.id)); + + // The server page only has the confirmed m1 + m5; the local-only ones the server never saw. + paginator.mergeNewestPage([msg('m1', 1), msg('m5', 5)], { + candidateIds, + requestedLimit: 5, + }); + + expect(paginator.getItem('sending')).toBeDefined(); + expect(paginator.getItem('failed')).toBeDefined(); + expect(paginator.getItem('err')).toBeDefined(); + expect(paginator.getItem('m1')).toBeDefined(); + expect(paginator.getItem('m5')).toBeDefined(); + }); + + it('removes a hard-deleted confirmed message while keeping a co-located failed one', () => { + const loaded = [ + msg('m1', 1), + msg('failed', 2, { status: 'failed' }), + msg('m3', 3), + msg('m4', 4), + ]; + const paginator = loadHead(loaded); + const candidateIds = new Set(loaded.map((message) => message.id)); + + // m3 hard-deleted; the failed send was never on the server. Page: [m1, m4]. + paginator.mergeNewestPage([msg('m1', 1), msg('m4', 4)], { + candidateIds, + requestedLimit: 4, + }); + + expect(paginator.getItem('m3')).toBeUndefined(); + expect(paginator.getItem('failed')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'failed', 'm4']); + }); + + // ── Empty page: whole channel emptied (or a missed truncate) ───────────────────────────────── + + it('empties the list when the channel returns no messages (with a snapshot)', () => { + const loaded = [msg('m1', 1), msg('m2', 2), msg('m3', 3)]; + const paginator = loadHead(loaded); + const candidateIds = new Set(loaded.map((message) => message.id)); + + paginator.mergeNewestPage([], { candidateIds, requestedLimit: 3 }); + + expect(paginator.items).toEqual([]); + expect(paginator.lastMessage).toBeNull(); + }); + + it('does NOT blank on an empty page without a snapshot (safe default)', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + paginator.mergeNewestPage([]); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + }); + + it('keeps a live arrival on an empty page (only snapshot ids are removed)', () => { + const loadedBefore = [msg('m1', 1), msg('m2', 2)]; + const paginator = loadHead(loadedBefore); + const candidateIds = new Set(loadedBefore.map((message) => message.id)); + + // A live message arrives during the fetch, then the (stale) empty page comes back. + paginator.ingestItem(msg('m3', 3)); + paginator.mergeNewestPage([], { candidateIds, requestedLimit: 2 }); + + expect(paginator.getItem('m1')).toBeUndefined(); + expect(paginator.getItem('m2')).toBeUndefined(); + expect(paginator.getItem('m3')).toBeDefined(); + expect(ids(paginator)).toEqual(['m3']); + }); + + // ── Structural guards preserved ────────────────────────────────────────────────────────────── + + it('does not reconcile on a disjoint reset (the rebuilt window is already authoritative)', () => { + const loaded = [msg('m1', 1), msg('m2', 2), msg('m3', 3)]; + const paginator = loadHead(loaded); + const candidateIds = new Set(loaded.map((message) => message.id)); + + // A fully-disjoint newest window (100+ arrived). Rebuild replaces the loaded set; no extra prune. + paginator.mergeNewestPage([msg('m10', 10), msg('m11', 11), msg('m12', 12)], { + candidateIds, + requestedLimit: 3, + }); + + expect(ids(paginator)).toEqual(['m10', 'm11', 'm12']); + }); + + it('does not reconcile when the caller is viewing a separate older window', () => { + const paginator = new MessagePaginator({ + channel, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + }); + paginator.ingestPage({ + page: [msg('m8', 8), msg('m9', 9), msg('m10', 10)], + isHead: true, + isTail: false, + setActive: true, + }); + paginator.ingestPage({ + page: [msg('m1', 1), msg('m2', 2), msg('m3', 3)], + isHead: false, + isTail: false, + setActive: true, + }); + + // Active window is the older [m1,m2,m3]; the head holds m8,m9,m10 (m9 "deleted" server-side). + paginator.mergeNewestPage([msg('m8', 8), msg('m10', 10)], { + candidateIds: new Set(['m8', 'm9', 'm10']), + requestedLimit: 3, + }); + + // Skipped entirely: the older window is preserved and the head ghost m9 is untouched. + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + expect(paginator.getItem('m9')).toBeDefined(); + }); + + it('is idempotent — a second reconcile against the same page removes nothing more', () => { + const loaded = [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)]; + const paginator = loadHead(loaded); + + paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3), msg('m4', 4)], { + candidateIds: new Set(loaded.map((message) => message.id)), + requestedLimit: 4, + }); + expect(ids(paginator)).toEqual(['m1', 'm3', 'm4']); + + paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3), msg('m4', 4)], { + candidateIds: new Set(['m1', 'm3', 'm4']), + requestedLimit: 4, + }); + expect(ids(paginator)).toEqual(['m1', 'm3', 'm4']); + }); + + // ── The "100 point" is data-driven, never hardcoded: reconcile only within the returned page ── + + it('never reconciles messages older than what the page returns, even on an over-request (clamp)', () => { + // 105 loaded (whole channel). The server caps its response at 100 (the newest 100). We requested + // more (105) but only 100 come back — that shortfall must NOT be read as "reached the channel + // start", or the 5 oldest (beyond the page) would be wrongly deleted. + const all = Array.from({ length: 105 }, (_, i) => + msg(`msg-${String(i).padStart(3, '0')}`, i), + ); + const paginator = loadHead(all, { isTail: true }); + const page = all.slice(5); // the newest 100 — the server's capped response + + paginator.mergeNewestPage(page, { + candidateIds: new Set(all.map((message) => message.id)), + requestedLimit: 105, + }); + + // All 105 kept: nothing was actually deleted, and the 5 oldest are beyond the page's reach. + expect(paginator.items?.length).toBe(105); + expect(paginator.getItem('msg-000')).toBeDefined(); + expect(paginator.getItem('msg-004')).toBeDefined(); + }); + + it('reconciles a deletion inside the returned page while keeping messages beyond it', () => { + const all = Array.from({ length: 105 }, (_, i) => + msg(`msg-${String(i).padStart(3, '0')}`, i), + ); + const paginator = loadHead(all, { isTail: true }); + // msg-050 hard-deleted; the server's newest 100 now reaches one further back. + const survivors = all.filter((message) => message.id !== 'msg-050'); + const page = survivors.slice(survivors.length - 100); + + paginator.mergeNewestPage(page, { + candidateIds: new Set(all.map((message) => message.id)), + requestedLimit: 105, + }); + + expect(paginator.getItem('msg-050')).toBeUndefined(); // within-page delete removed + expect(paginator.getItem('msg-000')).toBeDefined(); // beyond the page → kept + }); + + // ── Offline DB is kept in lockstep, entirely from the LLC (no SDK orchestration) ───────────── + + it('mirrors reconciled ghosts into the offline DB in one batch (LLC-owned)', async () => { + const hardDeleteMessage = vi.fn().mockResolvedValue([]); + const executeSqlBatch = vi.fn().mockResolvedValue(undefined); + const channelWithOfflineDb = { + cid: 'channel-id', + getReplies: vi.fn(), + query: vi.fn(), + getClient: () => ({ offlineDb: { hardDeleteMessage, executeSqlBatch } }), + } as unknown as Channel; + const paginator = new MessagePaginator({ + channel: channelWithOfflineDb, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + }); + paginator.ingestPage({ + page: [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + isHead: true, + isTail: true, + setActive: true, + }); + + // m3 hard-deleted while offline: gone from the in-memory list AND from SQLite. + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m4', 4)]); + + expect(paginator.getItem('m3')).toBeUndefined(); + expect(hardDeleteMessage).toHaveBeenCalledWith({ id: 'm3', execute: false }); + // The per-id delete queries are collected (execute:false) and run as a single transaction. + await Promise.resolve(); + await Promise.resolve(); + expect(executeSqlBatch).toHaveBeenCalledTimes(1); + }); + + it('reconciles in-memory without error when offline support is disabled (no offlineDb)', () => { + // The default mock channel has no getClient/offlineDb → the DB purge is a guarded no-op. + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + expect(() => paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3)])).not.toThrow(); + expect(paginator.getItem('m2')).toBeUndefined(); + }); + }); + describe('trackLastMessage() / lastMessageAt', () => { let skipSystemMessages: boolean; let trackingChannel: Channel; From 4b34488108d5e47d2da2e6d78cdb8697ab31fa7b Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Tue, 11 Aug 2026 15:03:32 +0200 Subject: [PATCH 02/16] chore: move api to offline db api --- src/offline-support/offline_support_api.ts | 28 ++++++++++ .../paginators/MessageIntervalPaginator.ts | 9 ++-- .../offline_support_api.test.ts | 54 +++++++++++++++++++ .../paginators/MessagePaginator.test.ts | 16 +++--- 4 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/offline-support/offline_support_api.ts b/src/offline-support/offline_support_api.ts index ef478595b..29b9098f3 100644 --- a/src/offline-support/offline_support_api.ts +++ b/src/offline-support/offline_support_api.ts @@ -746,6 +746,34 @@ export abstract class AbstractOfflineDB implements OfflineDBApi { return queries; }; + /** + * Hard-delete a set of messages by id in a single transaction. A convenience over N individual + * {@link hardDeleteMessage} calls: collects each delete's queries (`execute: false`) and runs them + * as one batch. Used e.g. by destructive reconciliation on reconnect to mirror a set of in-memory + * removals into the DB. No-op for an empty set. + * + * @param payload.ids - The ids of the messages to hard-delete. + * @param payload.execute - Whether to immediately execute the operation (optional, defaults to `true`). + */ + public hardDeleteMessages = async ({ + ids, + execute = true, + }: { + ids: string[]; + execute?: boolean; + }) => { + if (!ids.length) return []; + const queries = ( + await Promise.all(ids.map((id) => this.hardDeleteMessage({ id, execute: false }))) + ).flat(); + + if (execute) { + await this.executeSqlBatch(queries); + } + + return queries; + }; + /** * A utility method to handle read events. It will calculate the state of the reads if * present in the event, or optionally rely on the hard override in unreadMessages. diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index c74a8049e..0e63628a2 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -799,11 +799,10 @@ export class MessageIntervalPaginator extends BasePaginator< private purgeReconciledFromOfflineDb(ids: string[]) { const offlineDb = this.channel.getClient?.()?.offlineDb; if (!offlineDb) return; - Promise.all(ids.map((id) => offlineDb.hardDeleteMessage({ id, execute: false }))) - .then((queryBatches) => offlineDb.executeSqlBatch(queryBatches.flat())) - .catch(() => { - // best-effort persistence cleanup — see doc comment; the live list is already correct. - }); + // The offline DB owns the batching (single transaction); we just hand it the reconciled ids. + offlineDb.hardDeleteMessages({ ids }).catch(() => { + // best-effort persistence cleanup — see doc comment; the live list is already correct. + }); } protected resolveUnreadBoundaryIdsByTimestamp = ({ diff --git a/test/unit/offline-support/offline_support_api.test.ts b/test/unit/offline-support/offline_support_api.test.ts index 10aec4e9c..51c4caa0d 100644 --- a/test/unit/offline-support/offline_support_api.test.ts +++ b/test/unit/offline-support/offline_support_api.test.ts @@ -816,6 +816,60 @@ describe('OfflineSupportApi', () => { }); }); + describe('hardDeleteMessages', () => { + beforeEach(() => { + offlineDb.hardDeleteMessage.mockResolvedValue([['DELETE hard']]); + offlineDb.executeSqlBatch.mockResolvedValue(undefined); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + it('hard deletes every id and runs all queries as a single batch', async () => { + const result = await offlineDb.hardDeleteMessages({ ids: ['a', 'b'] }); + + expect(offlineDb.hardDeleteMessage).toHaveBeenCalledTimes(2); + expect(offlineDb.hardDeleteMessage).toHaveBeenCalledWith({ + id: 'a', + execute: false, + }); + expect(offlineDb.hardDeleteMessage).toHaveBeenCalledWith({ + id: 'b', + execute: false, + }); + // Each id's queries are collected (execute:false) and flushed in one transaction. + expect(offlineDb.executeSqlBatch).toHaveBeenCalledTimes(1); + expect(offlineDb.executeSqlBatch).toHaveBeenCalledWith([ + ['DELETE hard'], + ['DELETE hard'], + ]); + expect(result).toEqual([['DELETE hard'], ['DELETE hard']]); + }); + + it('returns the collected queries without executing them when execute is false', async () => { + const result = await offlineDb.hardDeleteMessages({ + ids: ['a'], + execute: false, + }); + + expect(offlineDb.hardDeleteMessage).toHaveBeenCalledWith({ + id: 'a', + execute: false, + }); + expect(offlineDb.executeSqlBatch).not.toHaveBeenCalled(); + expect(result).toEqual([['DELETE hard']]); + }); + + it('is a no-op for an empty id set (touches neither the delete nor the batch)', async () => { + const result = await offlineDb.hardDeleteMessages({ ids: [] }); + + expect(offlineDb.hardDeleteMessage).not.toHaveBeenCalled(); + expect(offlineDb.executeSqlBatch).not.toHaveBeenCalled(); + expect(result).toEqual([]); + }); + }); + describe('handleRead', () => { let readEvent: Event; diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index bd23ae5a6..a79bddadd 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2343,14 +2343,13 @@ describe('MessagePaginator', () => { // ── Offline DB is kept in lockstep, entirely from the LLC (no SDK orchestration) ───────────── - it('mirrors reconciled ghosts into the offline DB in one batch (LLC-owned)', async () => { - const hardDeleteMessage = vi.fn().mockResolvedValue([]); - const executeSqlBatch = vi.fn().mockResolvedValue(undefined); + it('mirrors reconciled ghosts into the offline DB via the DB batch API (LLC-owned)', () => { + const hardDeleteMessages = vi.fn().mockResolvedValue([]); const channelWithOfflineDb = { cid: 'channel-id', getReplies: vi.fn(), query: vi.fn(), - getClient: () => ({ offlineDb: { hardDeleteMessage, executeSqlBatch } }), + getClient: () => ({ offlineDb: { hardDeleteMessages } }), } as unknown as Channel; const paginator = new MessagePaginator({ channel: channelWithOfflineDb, @@ -2365,15 +2364,12 @@ describe('MessagePaginator', () => { setActive: true, }); - // m3 hard-deleted while offline: gone from the in-memory list AND from SQLite. + // m3 hard-deleted while offline: gone from the in-memory list AND from SQLite. The paginator + // just hands the reconciled ids to the DB's batch helper — it owns the transaction. paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m4', 4)]); expect(paginator.getItem('m3')).toBeUndefined(); - expect(hardDeleteMessage).toHaveBeenCalledWith({ id: 'm3', execute: false }); - // The per-id delete queries are collected (execute:false) and run as a single transaction. - await Promise.resolve(); - await Promise.resolve(); - expect(executeSqlBatch).toHaveBeenCalledTimes(1); + expect(hardDeleteMessages).toHaveBeenCalledWith({ ids: ['m3'] }); }); it('reconciles in-memory without error when offline support is disabled (no offlineDb)', () => { From 7be972e6de2832149cec98d36b0ed0a2786f2e03 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 01:20:27 +0200 Subject: [PATCH 03/16] feat: generalize merging the newest page and destructive reconciliation --- src/channel.ts | 72 +++- src/client.ts | 2 + .../paginators/MessageIntervalPaginator.ts | 76 +++- src/pagination/paginators/MessagePaginator.ts | 1 + test/unit/channel.test.js | 143 +++++++ .../paginators/MessagePaginator.test.ts | 371 +++++++++++++++++- test/unit/threads.test.ts | 65 +++ 7 files changed, 714 insertions(+), 16 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index 0a023e8ec..6753a623d 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -17,7 +17,6 @@ import { } from './utils'; import type { StreamChat } from './client'; import { chatLoggerSystem } from './logger'; -import { DEFAULT_QUERY_CHANNEL_MESSAGE_LIST_PAGE_SIZE } from './constants'; import type { AIState, APIResponse, @@ -175,6 +174,8 @@ export class Channel extends ChannelApi { lastTypingEvent: Date | null; isTyping: boolean; disconnected: boolean; + /** Re-entrancy guard for {@link Channel.reload} (mirrors Thread.reload's isLoading guard). */ + private _reloading = false; push_preferences?: Gen_ChannelPushPreferencesResponse; public readonly configState = new StateStore({}); public readonly messageComposer: MessageComposer; @@ -1297,6 +1298,46 @@ export class Channel extends ChannelApi { return state; } + /** + * Re-watch the channel and refresh its FULL loaded message window — the channel analog of + * {@link Thread.reload}. Used on reconnect to catch up AND reconcile hard deletes that happened + * while offline: a hard delete reaches other clients via no event, so an offline client only learns + * of it by diffing the re-queried page. + * + * This is intentionally thin: it only re-issues `watch()` with a limit sized to the loaded window + * (`items.length`, so the whole loaded window is refreshed — not the smaller channel-list page). The + * actual fold + destructive reconciliation happens inside `query()` → `seedFirstPageSync` + * (the same path the channel-list re-hydrate and React's `recoverState` use), driven by the loaded-id + * snapshot `query()` captures before its await. Owning that single path is what lets the SDK stop + * passing the reconciliation window/snapshot itself. + * + * Preserves failed (unsent) messages: an overlap merge keeps them (the reconcile's provenance guard + * never prunes a non-server message); only a disjoint rebuild can drop them, so any that actually + * fell out are re-ingested below. + */ + async reload() { + if (this._reloading || (!this.initialized && !this.offlineMode)) return; + this._reloading = true; + try { + const paginator = this.messagePaginator; + // Captured BEFORE the await: request our full loaded window (not the list's smaller page), and + // remember failed (unsent) messages so a disjoint rebuild does not silently drop them. + const requestedLimit = paginator.items?.length || paginator.pageSize; + const failedBefore = (paginator.items ?? []).filter( + (message) => message.status === 'failed', + ); + + await this.watch({ messages: { limit: requestedLimit } }); + this.offlineMode = false; + + for (const failed of failedBefore) { + if (!paginator.getItem(failed.id)) paginator.ingestItem(failed); + } + } finally { + this._reloading = false; + } + } + /** * Stops watching the channel. * @@ -1482,6 +1523,24 @@ export class Channel extends ChannelApi { options: ChannelGetOrCreateRequest = {}, messageSetToAddToIfDoesNotExist: MessageSetType = 'current', ) { + // Snapshot the loaded message ids BEFORE the network await, for a latest-window (re)seed only. + // When this query re-seeds an already-loaded window (reconnect / re-hydrate), seedFirstPageSync + // reconciles the fresh page against what is loaded, and the snapshot lets it tell an offline + // hard-delete (in the snapshot, absent from the page) from a message that arrives live + // during the fetch (not in the snapshot). Captured here — the only place with the pre-await state — + // so callers (channel.reload, watch) need not thread it. Empty on a cold open, so it is harmless. + const candidateIds = + messageSetToAddToIfDoesNotExist === 'latest' + ? new Set(this.messagePaginator.items?.map((message) => message.id) ?? []) + : undefined; + + // The INITIAL channel-open query honors the paginator's OWN pageSize (light on native, 25) rather + // than the server's larger default — opening loads the same page size it paginates by. A caller + // that already knows how much to fetch passes an explicit messages.limit, which is respected as-is: + // a reconnect/re-hydrate sizes it to the loaded window (channel.reload → items.length), and + // pagination/around pass their own cursors + limit. + const requestedPageSize = options?.messages?.limit ?? this.messagePaginator.pageSize; + // Make sure we wait for the connect promise if there is a pending one await this.getClient().wsPromise; @@ -1489,6 +1548,13 @@ export class Channel extends ChannelApi { data: this._data, state: true, ...options, + // Ask the server for exactly the initial-open page size (not its default), so the loaded window + // matches the paginator's pageSize. Explicit messages (reconnect/around/pagination) pass through. + messages: + options?.messages ?? + (messageSetToAddToIfDoesNotExist === 'latest' + ? { limit: requestedPageSize } + : undefined), }; const state = this.id @@ -1542,8 +1608,6 @@ export class Channel extends ChannelApi { // latest-page open paths (watch/create) pass 'latest' — the paginator's own pagination queries // use 'current' and must not be reseeded as a first page here. if (messageSetToAddToIfDoesNotExist === 'latest' && Array.isArray(state.messages)) { - const requestedPageSize = - options?.messages?.limit ?? DEFAULT_QUERY_CHANNEL_MESSAGE_LIST_PAGE_SIZE; // Pass the query's message pagination options through: a channel can be opened AROUND a // message (id_around / created_at_around), in which case the fetched page is a jump window, // not the latest page — the paginator must reconcile it with jump semantics. @@ -1551,6 +1615,8 @@ export class Channel extends ChannelApi { state.messages.map(formatMessage), requestedPageSize, options?.messages, + // Re-seed of an already-loaded window folds + reconciles instead of blanking (see above). + { candidateIds, reconcile: true }, ); } diff --git a/src/client.ts b/src/client.ts index be680e1fd..e2e7d9c57 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1445,6 +1445,8 @@ export class StreamChat extends ChatApi { c.messagePaginator.seedFirstPageSync( channelState.messages.map(formatMessage), requestedPageSize, + undefined, + { reconcile: true }, ); } diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index 0e63628a2..6fbcbfb53 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -173,6 +173,28 @@ export type MergeNewestPageOptions = { candidateIds?: ReadonlySet; }; +/** + * Options for {@link MessageIntervalPaginator.seedFirstPageSync}, the synchronous channel-open seed. + */ +export type SeedFirstPageOptions = { + /** + * When true and the paginator has already been seeded once, fold the fresh page via + * {@link MessageIntervalPaginator.mergeNewestPage} — merge in place, reconcile offline hard-deletes, + * re-derive the tail cursor, rebuild on a disjoint window, and skip when jumped away from the head — + * instead of a plain first-page ingest. Lets callers (channel.reload, the channel-list hydrate, + * React's `recoverState`) share one reconciling seed path. Left false for the differently-sorted + * pinned list, and a no-op on a cold open (nothing loaded to fold). + */ + reconcile?: boolean; + /** + * Pre-fetch snapshot of loaded message ids (captured before the caller's network await), forwarded + * to {@link MergeNewestPageOptions.candidateIds} so a hard-deleted NEWEST message is reconciled + * without mistaking a message that arrived live during the fetch for a delete. Only consulted when + * {@link reconcile} is set and a window is already loaded. + */ + candidateIds?: ReadonlySet; +}; + /** * MessageIntervalPaginator allows configuring backend request sort, while keeping internal item ordering stable. * Filtering of ingested items is still limited to local predicates (`filterQueryResults`). @@ -430,12 +452,22 @@ export class MessageIntervalPaginator extends BasePaginator< messages: LocalMessage[], requestedPageSize: number, messagePaginationOptions?: MessagePaginationOptions, + options?: SeedFirstPageOptions, ) { const queryShape: MessageQueryShape = { ...messagePaginationOptions, limit: requestedPageSize, }; const isJump = this.isJumpQueryShape(queryShape); + + if (options?.reconcile && !isJump && typeof this.items !== 'undefined') { + this.mergeNewestPage(messages, { + candidateIds: options.candidateIds, + requestedLimit: requestedPageSize, + }); + return; + } + this.postQueryReconcile({ // A jump/around page spans both directions; a plain latest page paginates tailward (older). direction: isJump ? undefined : 'tailward', @@ -579,9 +611,11 @@ export class MessageIntervalPaginator extends BasePaginator< * 1. OVERLAP - the incoming page shares at least one id with the loaded head (fewer than a full * page is new). Merge in place: existing items are reconciled by id (edits, soft deletes), new * items are appended and every already loaded item (including older pages already paged in) is - * kept. `hasMoreTail`/`cursor.tailward` are left as-is so the page can be any size, so deriving - * "has older items" from its length would wrongly clear it while older items remain. Then - * destructive reconciliation ({@link reconcileLoadedAgainstPage}) removes any loaded message + * kept. The tail boundary (`hasMoreTail`/`cursor.tailward`) is taken from the MERGED interval: a + * partial page leaves it untouched, while a page reaching deeper than the loaded window (or a + * stale offline-DB cursor) re-anchors it to the true loaded oldest so "load older" keeps working — + * derived from the interval, never the page's length. Then destructive reconciliation + * ({@link reconcileLoadedAgainstPage}) removes any loaded message * that the authoritative page proves was hard-deleted while offline (see that method + the * {@link MergeNewestPageOptions} for the exact, safe window). * @@ -649,16 +683,46 @@ export class MessageIntervalPaginator extends BasePaginator< return; } - // Overlapping window: merge in place, preserving the older boundary. + // Overlapping window: merge in place, keeping every already-loaded (incl. older) item. const interval = this.ingestPage({ page, isHead: true, setActive: false }); if (!interval) return; + // Re-compute hasMoreTail from the FETCHED PAGE, not the merged interval. The interval's isTail is + // "sticky" — mergeTwoAnchoredIntervals ORs isTail — so a stale offline-DB window persisted as + // "complete" (isTail:true) keeps hasMoreTail=false through the merge, and "load older" stays dead. + // A full page (length == the limit we asked for) means older messages remain; a short page means we + // reached the channel start. Without a caller-supplied limit (a live-edit merge of unknown size) + // fall back to the interval's own flag. Pagination reads off STATE, so writing it here is what + // unblocks "load older"; a later executeQuery re-derives per page. Mirrors postQueryReconcile, + // which likewise derives this flag from the page rather than the interval. + const { requestedLimit } = options ?? {}; + const canDeriveTail = typeof requestedLimit === 'number'; + const reachedChannelStart = + canDeriveTail && + page.length < Math.min(requestedLimit, DEFAULT_CHANNEL_MESSAGE_LIST_PAGE_SIZE); + const hasMoreTail = canDeriveTail ? !reachedChannelStart : interval.hasMoreTail; + + // Correct the INTERVAL flags too, not just state. `intervalsOverlap` (the merge test in + // ingestPage) consults `interval.isTail`: a sticky `isTail:true` inherited from a "complete" + // offline-DB window makes ANY older page count as overlapping this head interval — so a far + // jump's load-older welds two pages-apart sets into one. Derive isTail from the page here like the + // state above; mirrors postQueryReconcile (`interval.isTail = hasMoreTail === false`). Only a + // genuine channel-start interval (no older messages) keeps isTail:true. + interval.hasMoreTail = hasMoreTail; + interval.isTail = hasMoreTail === false; + this.setActiveInterval(interval, { updateState: false }); this.state.partialNext({ items: this.intervalToItems(interval), - // The newest slice is loaded (head anchored), so after merging the head window there is - // nothing newer to load. hasMoreTail / cursor are deliberately preserved (see above). + // The newest slice is loaded (head anchored), so there is nothing newer to load. hasMoreHead: false, + hasMoreTail, + // Tailward = the oldest loaded id (interval item ids are created_at asc), null once we reached + // the channel start. + cursor: { + headward: null, + tailward: hasMoreTail ? (interval.itemIds[0] ?? null) : null, + }, }); // With the newest page merged in, drop any loaded message the page proves was hard-deleted. diff --git a/src/pagination/paginators/MessagePaginator.ts b/src/pagination/paginators/MessagePaginator.ts index 853a40391..b1b99a8d9 100644 --- a/src/pagination/paginators/MessagePaginator.ts +++ b/src/pagination/paginators/MessagePaginator.ts @@ -23,6 +23,7 @@ export type { MessagePaginatorSort, MessagePaginatorState, MessageQueryShape, + SeedFirstPageOptions, } from './MessageIntervalPaginator'; export { MessageIntervalPaginator } from './MessageIntervalPaginator'; diff --git a/test/unit/channel.test.js b/test/unit/channel.test.js index 999a3662f..6501405ae 100644 --- a/test/unit/channel.test.js +++ b/test/unit/channel.test.js @@ -3458,3 +3458,146 @@ describe('share location', () => { }); }); }); + +describe('Channel.query — initial page size', () => { + let client; + let channel; + + beforeEach(() => { + client = new StreamChat('apiKey'); + client.user = { id: 'user' }; + const channelResponse = generateChannel(); + channel = client.channel(channelResponse.channel.type, channelResponse.channel.id); + channel.initialized = true; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('honors the paginator pageSize for the INITIAL open, not the server default', async () => { + channel.messagePaginator.pageSize = 25; + const getOrCreate = vi + .spyOn(channel, 'getOrCreate') + .mockResolvedValue( + generateChannel({ channel: { id: channel.id, type: channel.type } }), + ); + + await channel.query({}, 'latest'); + + // The initial open asks the server for exactly pageSize messages (not its larger default). + expect(getOrCreate).toHaveBeenCalledWith( + expect.objectContaining({ messages: { limit: 25 } }), + ); + }); + + it('respects an explicit messages.limit (reconnect sizes it to the loaded window)', async () => { + channel.messagePaginator.pageSize = 25; + const getOrCreate = vi + .spyOn(channel, 'getOrCreate') + .mockResolvedValue( + generateChannel({ channel: { id: channel.id, type: channel.type } }), + ); + + // e.g. channel.reload → watch({ messages: { limit: items.length } }) — passed through as-is. + await channel.query({ messages: { limit: 80 } }, 'latest'); + + expect(getOrCreate).toHaveBeenCalledWith( + expect.objectContaining({ messages: { limit: 80 } }), + ); + }); +}); + +describe('Channel.reload', () => { + let client; + let channel; + + const at = (minute) => new Date(Date.UTC(2020, 0, 1, 0, minute, 0)); + // Messages need the channel cid so the main-list paginator's ingestItem filter ({ cid }) accepts them. + const msg = (id, minute, overrides = {}) => + generateMsg({ id, cid: channel.cid, date: at(minute), ...overrides }); + + beforeEach(() => { + client = new StreamChat('apiKey'); + client.user = { id: 'user' }; + const channelResponse = generateChannel(); + channel = client.channel(channelResponse.channel.type, channelResponse.channel.id); + channel.initialized = true; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('reconciles a message hard-deleted while offline and keeps one that arrived during the fetch', async () => { + // m3 (newest loaded) is the one hard-deleted while offline. + seedLatestWindow(channel, [msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + expect(channel.messagePaginator.items.map((m) => m.id)).toEqual(['m1', 'm2', 'm3']); + + // The fold + reconcile now lives in query() → seedFirstPageSync (shared with the channel-list + // re-hydrate and React's recoverState); reload() is just watch() with the full-window limit. + // query() snapshots the loaded ids BEFORE this fetch, so a brand-new message that lands via WS + // DURING it (below) — absent from the server page — must survive. This exercises the whole + // snapshot-before-await + reconcile chain end to end, not the paginator in isolation. + vi.spyOn(channel, 'getOrCreate').mockImplementation(async () => { + channel.messagePaginator.ingestItem(formatMessage(msg('m4', 4))); + return generateChannel({ + channel: { id: channel.id, type: channel.type }, + messages: [msg('m1', 1), msg('m2', 2)], + }); + }); + + await channel.reload(); + + // m3 (in the pre-fetch snapshot, absent from the page) removed; m4 (arrived after) kept. + expect(channel.messagePaginator.items.map((m) => m.id)).toEqual(['m1', 'm2', 'm4']); + }); + + it('requests the full loaded window (items.length), not the channel-list page size', async () => { + seedLatestWindow(channel, [msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + const watchSpy = vi + .spyOn(channel, 'watch') + .mockResolvedValue({ messages: [msg('m1', 1), msg('m2', 2), msg('m3', 3)] }); + + await channel.reload(); + + expect(watchSpy).toHaveBeenCalledWith({ messages: { limit: 3 } }); + }); + + it('preserves a failed (unsent) message that a disjoint rebuild would otherwise drop', async () => { + seedLatestWindow(channel, [ + msg('m1', 1), + msg('failed', 2, { status: 'failed' }), + msg('m3', 3), + ]); + // A page that shares no id with the loaded window is disjoint, so the fold rebuilds and discards + // local-only messages — reload must re-ingest the failed one so it is not lost. + vi.spyOn(channel, 'getOrCreate').mockImplementation(async () => + generateChannel({ + channel: { id: channel.id, type: channel.type }, + messages: [msg('n8', 8), msg('n9', 9)], + }), + ); + + await channel.reload(); + + expect(channel.messagePaginator.getItem('failed')).toBeDefined(); + }); + + it('ignores a re-entrant reload while one is already in flight', async () => { + seedLatestWindow(channel, [msg('m1', 1)]); + let resolveWatch; + const watchSpy = vi.spyOn(channel, 'watch').mockReturnValue( + new Promise((resolve) => { + resolveWatch = () => resolve({ messages: [msg('m1', 1)] }); + }), + ); + + const inFlight = channel.reload(); + await channel.reload(); // guarded — returns immediately, must not call watch again + resolveWatch(); + await inFlight; + + expect(watchSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index a79bddadd..12f2da25e 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -1735,20 +1735,20 @@ describe('MessagePaginator', () => { expect(paginator.items?.map((message) => message.id)).toEqual(['m1', 'm2', 'm3']); }); - it('preserves hasMoreTail / cursor.tailward when merging a partial newest window', () => { + it('keeps hasMoreTail and anchors the tail cursor to the loaded oldest when merging a partial newest window', () => { // Only the newest window is loaded and older items still exist (hasMoreTail true). Merging a // short page (fewer than pageSize) whose first item is the set's first item must NOT clear - // hasMoreTail: re-deriving it from this page's length would wrongly break "load older", so the - // merge preserves the existing hasMoreTail / cursor instead. + // hasMoreTail — re-deriving it from the page's LENGTH would wrongly break "load older". The tail + // boundary is taken from the MERGED interval, so hasMoreTail stays true and the cursor anchors to + // the loaded oldest (m1) — the correct "load older" anchor. const { paginator, m1, m2 } = setupLoadedHead({ isTail: false }); expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); - const tailwardBefore = paginator.state.getLatestValue().cursor?.tailward; const editedM3 = m('m3', '03', { text: 'edited' }); paginator.mergeNewestPage([m1, m2, editedM3]); expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); - expect(paginator.state.getLatestValue().cursor?.tailward).toBe(tailwardBefore); + expect(paginator.state.getLatestValue().cursor?.tailward).toBe('m1'); expect(paginator.getItem('m3')?.text).toBe('edited'); }); @@ -1817,7 +1817,6 @@ describe('MessagePaginator', () => { it('treats a window sharing only the loaded newest id as OVERLAP, not disjoint (boundary)', () => { const { paginator, m3 } = setupLoadedHead({ isTail: false }); - const tailwardBefore = paginator.state.getLatestValue().cursor?.tailward; // Exactly one shared id (the loaded newest, m3): the minimal-overlap boundary. This must merge // (append m4/m5, keep older loadable), NOT reset to the window. paginator.mergeNewestPage([m3, m('m4', '04'), m('m5', '05')]); @@ -1831,7 +1830,8 @@ describe('MessagePaginator', () => { ]); expect(paginator.itemIntervals).toHaveLength(1); expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); - expect(paginator.state.getLatestValue().cursor?.tailward).toBe(tailwardBefore); + // The tail cursor anchors to the loaded oldest (m1), derived from the merged interval. + expect(paginator.state.getLatestValue().cursor?.tailward).toBe('m1'); }); // Builds the "jumped away" shape: the newest slice is loaded as one interval, and a separate @@ -2380,6 +2380,363 @@ describe('MessagePaginator', () => { }); }); + // seedFirstPageSync is the synchronous channel-open seed (Channel.query / hydrateActiveChannels). + // With `options.reconcile` it doubles as the reconnect / re-hydrate fold: over an already-loaded + // window it delegates to mergeNewestPage (merge + destructive reconcile + disjoint rebuild), whose + // internals are covered above — these tests pin only the ROUTING decision (which branch it picks). + describe('seedFirstPageSync() — reconcile routing', () => { + const msg = (id: string, minute: number, overrides: Partial = {}) => + createMessage({ + cid: 'channel-id', + id, + created_at: new Date(Date.UTC(2020, 0, 1, 0, minute, 0)).toISOString(), + ...overrides, + }); + + // The plain-seed branch runs seedUnreadSnapshot (reads getClient().user); give the channel a + // benign client with no current user so it no-ops instead of throwing on the bare mock. + const reconcileChannel = { + cid: 'channel-id', + getReplies: vi.fn(), + query: vi.fn(), + getClient: () => ({ user: undefined }), + } as unknown as Channel; + + const makePaginator = () => + new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + }); + + const loadHead = (messages: LocalMessage[]) => { + const paginator = makePaginator(); + paginator.ingestPage({ + page: messages, + isHead: true, + isTail: false, + setActive: true, + }); + return paginator; + }; + + const ids = (paginator: MessagePaginator) => + paginator.items?.map((message) => message.id); + + it('REPRO(interval): reconnect re-establishes hasMoreTail over a stale "complete" INTERVAL (offline DB)', () => { + const paginator = makePaginator(); + // Offline-DB window persisted as "complete" — the INTERVAL itself has isTail=true / hasMoreTail + // false, even though the channel has older messages the cache never held. (This is what my + // earlier state-only corruption failed to reproduce.) + paginator.ingestPage({ + page: [msg('m1', 1), msg('m2', 2), msg('m3', 3)], + isHead: true, + isTail: true, + setActive: true, + }); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); + + // Reconnect fetches a FULL page (requestedLimit == page length) → older messages remain, so + // hasMoreTail must be RE-COMPUTED from the page (not read off the stale interval flag). + paginator.seedFirstPageSync( + [msg('m1', 1), msg('m2', 2), msg('m3', 3)], + 3, + undefined, + { + reconcile: true, + }, + ); + + expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); + expect(paginator.state.getLatestValue().cursor?.tailward).toBe('m1'); + }); + + it('JOURNEY: a reconnect re-seed over a stale window keeps "load older" working end-to-end', async () => { + // The exact user flow that regressed: open a channel, its window is preloaded from the offline DB + // with a dead cursor, a reconnect re-seeds, then the user scrolls up. "Load older" must fetch and + // append the previous page — a component-level test (checking only which messages merged) missed + // this because the break was in the CURSOR, so drive the real executeQuery pagination here. + const older = [ + msg('m01', 1), + msg('m02', 2), + msg('m03', 3), + msg('m04', 4), + msg('m05', 5), + ]; + const doRequest = vi.fn().mockResolvedValue({ + items: older, + cursor: { tailward: 'm01', headward: 'm05' }, + }); + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: { doRequest }, + }); + + const head = [ + msg('m06', 6), + msg('m07', 7), + msg('m08', 8), + msg('m09', 9), + msg('m10', 10), + ]; + // Offline-DB window persisted as "complete" — the INTERVAL itself is isTail:true / hasMoreTail + // false (the real stale shape; corrupting only state would miss the bug). + paginator.ingestPage({ page: head, isHead: true, isTail: true, setActive: true }); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); + paginator.seedFirstPageSync(head, 5, undefined, { reconcile: true }); // reconnect re-seed + + // The user scrolls up. If the re-seed left the dead cursor, executeQuery no-ops (hasMoreTail + // false) and nothing loads; with the cursor re-derived it fetches and appends the older page. + await paginator.executeQuery({ direction: 'tailward' }); + + expect(doRequest).toHaveBeenCalled(); + expect(paginator.items?.map((m) => m.id)).toEqual([ + 'm01', + 'm02', + 'm03', + 'm04', + 'm05', + 'm06', + 'm07', + 'm08', + 'm09', + 'm10', + ]); + }); + + it('a reconnect re-seed while JUMPED AWAY does not weld the newest page into the active older window', () => { + const paginator = makePaginator(); + // Head window (newest), loaded on open. + paginator.ingestPage({ + page: [msg('m080', 80), msg('m090', 90), msg('m100', 100)], + isHead: true, + isTail: false, + setActive: true, + }); + // Jump to a far, DISJOINT older window (like clicking a quoted message in another set) — it + // becomes the active interval and is NOT the head. + paginator.ingestPage({ + page: [msg('m020', 20), msg('m021', 21), msg('m022', 22)], + isHead: false, + isTail: false, + setActive: true, + }); + expect(paginator.isActiveIntervalAtHead).toBe(false); + const before = paginator.items?.map((m) => m.id); + + // A reconnect re-seeds the newest page (channel.reload → watch → seedFirstPageSync). It must NOT + // weld the newest into the jumped-away window — the two message sets stay separate. + paginator.seedFirstPageSync( + [msg('m080', 80), msg('m090', 90), msg('m100', 100)], + 3, + undefined, + { reconcile: true }, + ); + + expect(paginator.isActiveIntervalAtHead).toBe(false); // still on the jumped window + expect(paginator.items?.map((m) => m.id)).toEqual(before); // unchanged — no weld + }); + + it('AUDIT: disjoint reconnect rebuilds to the fresh page instead of welding across the gap', () => { + const paginator = loadHead([msg('m01', 1), msg('m02', 2), msg('m03', 3)]); + // 100+ new arrived while offline → the fetched newest page shares NO id with the loaded window. + paginator.seedFirstPageSync( + [msg('m10', 10), msg('m11', 11), msg('m12', 12)], + 3, + undefined, + { + reconcile: true, + }, + ); + // Must NOT weld m10..m12 across the gap into m01..m03 (which hides m04..m09 with no way to reach + // them). Rebuild to the fresh page so scrolling up reloads the gap contiguously. + expect(paginator.items?.map((m) => m.id)).toEqual(['m10', 'm11', 'm12']); + }); + + it('AUDIT: an empty reconnect page (no snapshot) does not blank the loaded window', () => { + const paginator = loadHead([msg('m01', 1), msg('m02', 2), msg('m03', 3)]); + // A transient empty page on reconnect must not wipe the list on its own. + paginator.seedFirstPageSync([], 3, undefined, { reconcile: true }); + expect(paginator.items?.map((m) => m.id)).toEqual(['m01', 'm02', 'm03']); + }); + + it('AUDIT e2e: after a disjoint rebuild, "load older" reloads the gap, not the discarded stale window', async () => { + const gap = [msg('m175', 175), msg('m176', 176), msg('m177', 177)]; + const doRequest = vi.fn().mockResolvedValue({ + items: gap, + cursor: { tailward: 'm175', headward: 'm177' }, + }); + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: { doRequest }, + }); + // The newest window loaded when the user went offline. + paginator.ingestPage({ + page: [msg('m078', 78), msg('m079', 79), msg('m080', 80)], + isHead: true, + isTail: false, + setActive: true, + }); + // Reconnect: 100+ new arrived, so the fetched newest page is DISJOINT from the loaded window. + paginator.seedFirstPageSync( + [msg('m178', 178), msg('m179', 179), msg('m180', 180)], + 3, + undefined, + { + reconcile: true, + }, + ); + // Scroll up: the rebuilt window must reload the gap contiguously — the stale m078..m080 are gone, + // not welded in with the in-between messages hidden. + await paginator.executeQuery({ direction: 'tailward' }); + const ids = paginator.items?.map((m) => m.id); + expect(ids).not.toContain('m078'); + expect(ids).toEqual(['m175', 'm176', 'm177', 'm178', 'm179', 'm180']); + }); + + it('a jump to a far disjoint message stays SEPARATE from the latest, even through a re-seed', async () => { + const around = [msg('m20', 20), msg('m21', 21), msg('m22', 22)]; + const doRequest = vi.fn().mockResolvedValue({ + items: around, + cursor: { tailward: 'm20', headward: 'm22' }, + }); + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: { doRequest }, + }); + paginator.ingestPage({ + page: [msg('m80', 80), msg('m90', 90), msg('m100', 100)], + isHead: true, + isTail: false, + setActive: true, + }); + await paginator.jumpToMessage('m21'); + // A latest-window re-seed (what watch() → seedFirstPageSync fires) must NOT weld the jumped + // window into the latest — mergeNewestPage skips because the head is not the active interval. + paginator.seedFirstPageSync( + [msg('m80', 80), msg('m90', 90), msg('m100', 100)], + 3, + undefined, + { + reconcile: true, + }, + ); + expect(paginator.items?.map((m) => m.id)).toEqual(['m20', 'm21', 'm22']); + expect(paginator.itemIntervals.length).toBe(2); + }); + + it('reconciling seed clears a sticky isTail so a far older page cannot weld across the gap', () => { + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: {}, + }); + // An offline-DB latest window rehydrated as "complete" — isTail:true even though older + // messages exist on the server (the stale offline window). This is the flag intervalsOverlap + // consults to decide a merge. + paginator.ingestPage({ + page: [msg('m90', 90), msg('m95', 95), msg('m100', 100)], + isHead: true, + isTail: true, + setActive: true, + }); + expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(true); + + // The reconciling seed at the derived page size (a FULL page => older messages remain) must + // clear the sticky isTail — not just state's hasMoreTail. + paginator.seedFirstPageSync( + [msg('m90', 90), msg('m95', 95), msg('m100', 100)], + 3, + undefined, + { reconcile: true }, + ); + expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(false); + expect(paginator.hasMoreTail).toBe(true); + + // Jump to a far OLDER window as a separate interval. + const island = paginator.ingestPage({ + page: [msg('m10', 10), msg('m11', 11), msg('m12', 12)], + isHead: false, + isTail: false, + setActive: true, + }); + expect(paginator.itemIntervals.length).toBe(2); + + // Load older from the island: a page older than it, nowhere near the latest window. With a + // sticky isTail on the latest, intervalsOverlap would (wrongly) treat this as overlapping the + // latest and weld the two pages-apart sets into one. + paginator.ingestPage({ + page: [msg('m7', 7), msg('m8', 8), msg('m9', 9)], + isTail: false, + setActive: false, + targetIntervalId: island?.id, + }); + + // Stays two separate intervals — the older page merges only into the island. + expect(paginator.itemIntervals.length).toBe(2); + }); + + it('reconcile + already-loaded: folds the fresh page and drops a within-span hard-delete', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + // m2 hard-deleted while offline; the re-seed's authoritative page comes back without it. + paginator.seedFirstPageSync([msg('m1', 1), msg('m3', 3)], 3, undefined, { + reconcile: true, + }); + expect(ids(paginator)).toEqual(['m1', 'm3']); + expect(paginator.getItem('m2')).toBeUndefined(); + }); + + it('reconcile + snapshot: drops a trailing ghost while keeping a message that arrived during the fetch', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + // A live message lands AFTER the pre-fetch snapshot was taken (so it is not in candidateIds). + paginator.ingestItem(msg('m4', 4)); + const candidateIds = new Set(['m1', 'm2', 'm3']); + // The page (missing m3 — the hard-deleted newest — and predating m4) is the server truth. + paginator.seedFirstPageSync([msg('m1', 1), msg('m2', 2)], 3, undefined, { + reconcile: true, + candidateIds, + }); + // m3 removed (in the snapshot, absent from the page, at the top edge); m4 kept (a live arrival). + expect(ids(paginator)).toEqual(['m1', 'm2', 'm4']); + }); + + it('reconcile on a cold (never-seeded) paginator: plain-seeds the page', () => { + const paginator = makePaginator(); + expect(paginator.items).toBeUndefined(); + paginator.seedFirstPageSync([msg('m1', 1), msg('m2', 2)], 25, undefined, { + reconcile: true, + }); + expect(ids(paginator)).toEqual(['m1', 'm2']); + }); + + it('WITHOUT the reconcile flag: plain-seeds and never reconciles (the pinned-list contract)', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + // Same missing-m2 page, but no reconcile flag → additive seed; m2 is NOT removed. + paginator.seedFirstPageSync([msg('m1', 1), msg('m3', 3)], 3); + expect(paginator.getItem('m2')).toBeDefined(); + }); + + it('reconcile + a jump/around re-seed: applies jump semantics, never reconciles the latest window', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + // An around open is not the latest window, so the loaded messages must not be reconciled away. + paginator.seedFirstPageSync( + [msg('m5', 5), msg('m6', 6)], + 25, + { id_around: 'm5' }, + { + reconcile: true, + }, + ); + expect(paginator.getItem('m1')).toBeDefined(); + expect(paginator.getItem('m2')).toBeDefined(); + expect(paginator.getItem('m3')).toBeDefined(); + }); + }); + describe('trackLastMessage() / lastMessageAt', () => { let skipSystemMessages: boolean; let trackingChannel: Channel; diff --git a/test/unit/threads.test.ts b/test/unit/threads.test.ts index 06427b0a9..12d5e10c1 100644 --- a/test/unit/threads.test.ts +++ b/test/unit/threads.test.ts @@ -441,6 +441,35 @@ describe('Threads 2.0', () => { expect(repliesOf(thread).map((reply) => reply.id)).to.include(failedMessage.id); }); + it('re-derives a paginatable reply cursor over a stale window (Thread.reload stays paginatable offline)', () => { + const existingReply = generateMsg({ + parent_id: parentMessageResponse.id, + created_at: '2020-01-01T00:00:00.000Z', + }) as MessageResponse; + // Head-anchored, older replies still to load (reply_count > loaded). + const thread = createTestThread({ + latest_replies: [existingReply], + reply_count: 10, + }); + // Simulate a reply window preloaded with a stale/"complete" cursor (offline DB): "load older + // replies" is dead if the reconnect hydrate PRESERVES it instead of re-deriving. + thread.messagePaginator.state.partialNext({ + hasMoreTail: false, + cursor: { tailward: null, headward: null }, + }); + expect(thread.messagePaginator.state.getLatestValue().hasMoreTail).to.be.false; + + // Reconnect hydrate (Thread.reload → hydrateState → mergeNewestPage) must RE-DERIVE the cursor + // from the merged reply window so pagination works again. + const hydrationThread = createTestThread({ + latest_replies: [existingReply], + reply_count: 10, + }); + thread.hydrateState(hydrationThread); + + expect(thread.messagePaginator.state.getLatestValue().hasMoreTail).to.be.true; + }); + it('merges the incoming newest reply window into the reply paginator', () => { const existingReply = generateMsg({ parent_id: parentMessageResponse.id, @@ -584,6 +613,42 @@ describe('Threads 2.0', () => { expect(stub.secondCall.args[0]?.reply_limit).to.equal(7); expect(loadedThread.messagePaginator.pageSize).to.not.equal(7); }); + + it('removes a reply hard-deleted while offline and keeps one that arrived during the fetch', async () => { + // End-to-end through the REAL reload orchestration (not a hand-built snapshot): this is what + // proves the snapshot-before-await guarantee — the thing the paginator-level tests assume. + const r1 = makeReply({ id: 'r1', created_at: '2020-01-01T00:00:01.000Z' }); + const r2 = makeReply({ id: 'r2', created_at: '2020-01-01T00:00:02.000Z' }); + // r3 is the newest loaded reply — hard-deleted by someone else while we were offline. + const r3 = makeReply({ id: 'r3', created_at: '2020-01-01T00:00:03.000Z' }); + const thread = createTestThread({ + latest_replies: [r1, r2, r3], + reply_count: 3, + }); + expect(repliesOf(thread).map((reply) => reply.id)).to.eql(['r1', 'r2', 'r3']); + + // A brand-new reply that lands via WS DURING the reload fetch — after reload() snapshots the + // loaded ids, before hydrateState runs. Like the r3 ghost it is absent from the server page, + // so a naive "loaded − serverPage" would wrongly drop it; the pre-fetch snapshot must save it. + const r4 = makeReply({ id: 'r4', created_at: '2020-01-01T00:00:04.000Z' }); + + // The server's authoritative page (computed before r4 existed) has r3 hard-deleted, no r4. + const hydrationThread = createTestThread({ + latest_replies: [r1, r2], + reply_count: 2, + }); + + sinon.stub(client, 'getThreadAndHydrate').callsFake(async () => { + thread.messagePaginator.ingestItem(formatMessage(r4)); // live arrival during the await + return hydrationThread; + }); + + await thread.reload(); + + // r3 (in the pre-fetch snapshot, absent from the server page) → hard-delete, removed. + // r4 (arrived AFTER the snapshot) → not in the snapshot → kept. + expect(repliesOf(thread).map((reply) => reply.id)).to.eql(['r1', 'r2', 'r4']); + }); }); describe('deleteReplyLocally', () => { From 49509aaff1d160f2f47b1cc98fd34546cb631e1f Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 01:55:52 +0200 Subject: [PATCH 04/16] fix: execute reconciliation even if not at head interval --- .../paginators/MessageIntervalPaginator.ts | 17 ++++--- .../paginators/MessagePaginator.test.ts | 48 +++++++++++++++++-- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index 6fbcbfb53..b3f9c5d60 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -640,11 +640,16 @@ export class MessageIntervalPaginator extends BasePaginator< mergeNewestPage = (page: LocalMessage[], options?: MergeNewestPageOptions) => { const headInterval = this.itemIntervals[0] as Interval | undefined; if (!headInterval?.isHead) return; - // Only reconcile when the head is the interval currently in view. If the caller jumped to a - // separate (older) window, that window is active and the head is merely still-loaded underneath; - // reconciling would switch the view to the head and yank them to the newest. Skip to preserve - // their position (the newest page is picked up on scroll / a later load). - if (!this.isActiveInterval(headInterval)) return; + // If the caller jumped to a separate (older) window, that window is active and the head is merely + // still-loaded underneath. Don't MERGE the fresh page or switch the view (that would yank them to + // the newest) — but STILL prune offline hard-deletes out of the hidden head, otherwise returning to + // it later (scroll-to-latest) surfaces ghosts that were deleted while jumped away. Reconciliation + // targets the head interval regardless of which interval is active, and removing a hidden-head ghost + // leaves the active island's items untouched. + if (!this.isActiveInterval(headInterval)) { + this.reconcileLoadedAgainstPage(page, options); + return; + } if (!page?.length) { // Empty page: never blanks the list by itself. Only when the caller supplied a pre-fetch @@ -771,7 +776,7 @@ export class MessageIntervalPaginator extends BasePaginator< options?: MergeNewestPageOptions, ) { const headInterval = this.itemIntervals[0] as Interval | undefined; - if (!headInterval?.isHead || !this.isActiveInterval(headInterval)) return; + if (!headInterval?.isHead) return; const loadedIds = headInterval.itemIds; const candidateIds = options?.candidateIds; diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index 12f2da25e..d5c44c3ec 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2252,7 +2252,7 @@ describe('MessagePaginator', () => { expect(ids(paginator)).toEqual(['m10', 'm11', 'm12']); }); - it('does not reconcile when the caller is viewing a separate older window', () => { + it('reconciles the hidden head but preserves the view when the caller jumped to a separate older window', () => { const paginator = new MessagePaginator({ channel, itemIndex: new StoreBackedItemIndex({ @@ -2278,9 +2278,10 @@ describe('MessagePaginator', () => { requestedLimit: 3, }); - // Skipped entirely: the older window is preserved and the head ghost m9 is untouched. + // The view (the older window) is preserved — no yank to the head — but the hidden-head ghost m9 + // is still pruned, so returning to the head later (scroll-to-latest) won't surface it. expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); - expect(paginator.getItem('m9')).toBeDefined(); + expect(paginator.getItem('m9')).toBeUndefined(); }); it('is idempotent — a second reconcile against the same page removes nothing more', () => { @@ -2629,6 +2630,47 @@ describe('MessagePaginator', () => { expect(paginator.itemIntervals.length).toBe(2); }); + it('reconnect while jumped away still reconciles offline hard-deletes out of the hidden head', () => { + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: {}, + }); + // Head/latest window loaded and at head. + paginator.ingestPage({ + page: [msg('m90', 90), msg('m95', 95), msg('m100', 100)], + isHead: true, + isTail: false, + setActive: true, + }); + // Jump to a far older island — now jumped away; the head is loaded-but-hidden underneath. + paginator.ingestPage({ + page: [msg('m10', 10), msg('m11', 11), msg('m12', 12)], + isHead: false, + isTail: false, + setActive: true, + }); + expect(paginator.isActiveIntervalAtHead).toBe(false); + + // Reconnect: the fresh newest page proves m100 (bottom-most head message) was hard-deleted while + // offline — it is absent from the page and above the newest returned message, so only the + // pre-fetch snapshot can prune it. + const candidateIds = new Set(['m90', 'm95', 'm100', 'm10', 'm11', 'm12']); + paginator.mergeNewestPage([msg('m90', 90), msg('m95', 95)], { + candidateIds, + requestedLimit: 3, + }); + + // View is preserved — still on the island, unchanged. + expect(paginator.isActiveIntervalAtHead).toBe(false); + expect(paginator.items?.map((m) => m.id)).toEqual(['m10', 'm11', 'm12']); + // ...but the ghost is pruned from the hidden head, so scroll-to-latest won't surface it. + expect(paginator.getItem('m100')).toBeUndefined(); + expect((paginator.itemIntervals[0] as { itemIds: string[] }).itemIds).not.toContain( + 'm100', + ); + }); + it('reconciling seed clears a sticky isTail so a far older page cannot weld across the gap', () => { const paginator = new MessagePaginator({ channel: reconcileChannel, From 26ed33b19caf47b991342887b0893d745588e6b1 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 02:22:12 +0200 Subject: [PATCH 05/16] fix: use correct interval for candidate ids --- src/channel.ts | 2 +- .../paginators/MessagePaginator.test.ts | 53 +++++++++++++++---- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index 6753a623d..82682902e 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1531,7 +1531,7 @@ export class Channel extends ChannelApi { // so callers (channel.reload, watch) need not thread it. Empty on a cold open, so it is harmless. const candidateIds = messageSetToAddToIfDoesNotExist === 'latest' - ? new Set(this.messagePaginator.items?.map((message) => message.id) ?? []) + ? new Set(this.messagePaginator.headItems.map((message) => message.id)) : undefined; // The INITIAL channel-open query honors the paginator's OWN pageSize (light on native, 25) rather diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index d5c44c3ec..7a4c373d0 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2630,19 +2630,49 @@ describe('MessagePaginator', () => { expect(paginator.itemIntervals.length).toBe(2); }); - it('reconnect while jumped away still reconciles offline hard-deletes out of the hidden head', () => { + it('headItems is the hidden head window (not the active island) — the candidateIds source when jumped away', () => { const paginator = new MessagePaginator({ channel: reconcileChannel, itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), paginatorOptions: {}, }); - // Head/latest window loaded and at head. paginator.ingestPage({ page: [msg('m90', 90), msg('m95', 95), msg('m100', 100)], isHead: true, isTail: false, setActive: true, }); + paginator.ingestPage({ + page: [msg('m10', 10), msg('m11', 11), msg('m12', 12)], + isHead: false, + isTail: false, + setActive: true, + }); + // `items` follows the active view (the island) — the WRONG snapshot for reconciling the head... + expect(paginator.items?.map((m) => m.id)).toEqual(['m10', 'm11', 'm12']); + // ...`headItems` is the hidden head, which is what channel.query snapshots for candidateIds. + expect(paginator.headItems.map((m) => m.id)).toEqual(['m90', 'm95', 'm100']); + }); + + it('reconnect while jumped away prunes the whole trailing run from the hidden head', () => { + const paginator = new MessagePaginator({ + channel: reconcileChannel, + itemIndex: new StoreBackedItemIndex({ getEntityId: (m) => m.id }), + paginatorOptions: {}, + }); + // Head/latest window loaded and at head; m98,m99,m100 are the bottom-most (newest) messages. + paginator.ingestPage({ + page: [ + msg('m90', 90), + msg('m95', 95), + msg('m98', 98), + msg('m99', 99), + msg('m100', 100), + ], + isHead: true, + isTail: false, + setActive: true, + }); // Jump to a far older island — now jumped away; the head is loaded-but-hidden underneath. paginator.ingestPage({ page: [msg('m10', 10), msg('m11', 11), msg('m12', 12)], @@ -2652,23 +2682,24 @@ describe('MessagePaginator', () => { }); expect(paginator.isActiveIntervalAtHead).toBe(false); - // Reconnect: the fresh newest page proves m100 (bottom-most head message) was hard-deleted while - // offline — it is absent from the page and above the newest returned message, so only the - // pre-fetch snapshot can prune it. - const candidateIds = new Set(['m90', 'm95', 'm100', 'm10', 'm11', 'm12']); + // candidateIds is snapshotted by channel.query from `headItems` (the hidden head) — NOT `items` + // (the island). The whole trailing RUN m98,m99,m100 was hard-deleted offline; all three are above + // the newest survivor (m95), so only the head-derived snapshot can prune them. + const candidateIds = new Set(paginator.headItems.map((m) => m.id)); paginator.mergeNewestPage([msg('m90', 90), msg('m95', 95)], { candidateIds, requestedLimit: 3, }); - // View is preserved — still on the island, unchanged. + // View preserved (still on the island)... expect(paginator.isActiveIntervalAtHead).toBe(false); expect(paginator.items?.map((m) => m.id)).toEqual(['m10', 'm11', 'm12']); - // ...but the ghost is pruned from the hidden head, so scroll-to-latest won't surface it. + // ...and the ENTIRE trailing run is pruned from the hidden head — none surface on scroll-to-latest. + expect(paginator.getItem('m98')).toBeUndefined(); + expect(paginator.getItem('m99')).toBeUndefined(); expect(paginator.getItem('m100')).toBeUndefined(); - expect((paginator.itemIntervals[0] as { itemIds: string[] }).itemIds).not.toContain( - 'm100', - ); + const headIds = (paginator.itemIntervals[0] as { itemIds: string[] }).itemIds; + expect(headIds).toEqual(['m90', 'm95']); }); it('reconciling seed clears a sticky isTail so a far older page cannot weld across the gap', () => { From 2a06363790246d4bf19d908ecd56e1d3cc07f852 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 03:17:41 +0200 Subject: [PATCH 06/16] fix: cold load reconciliation --- src/client.ts | 16 ++++++++++++++-- test/unit/client.test.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index e2e7d9c57..fe3b53a3d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1332,6 +1332,12 @@ export class StreamChat extends ChatApi { options?: QueryChannelsRequest, stateOptions: ChannelStateOptions = {}, ): Promise { + const candidateIdsByCid = new Map>(); + for (const cid of Object.keys(this.activeChannels)) { + const head = this.activeChannels[cid]?.messagePaginator?.headItems; + if (head?.length) + candidateIdsByCid.set(cid, new Set(head.map((message) => message.id))); + } const queryChannelsResponse = await this.queryChannels(options); const channels = queryChannelsResponse.channels; @@ -1349,7 +1355,12 @@ export class StreamChat extends ChatApi { }); } - const hydratedChannels = this.hydrateActiveChannels(channels, stateOptions, options); + const hydratedChannels = this.hydrateActiveChannels( + channels, + stateOptions, + options, + candidateIdsByCid, + ); if (stateOptions.withResponse) { return { @@ -1405,6 +1416,7 @@ export class StreamChat extends ChatApi { channelsFromApi: ChannelStateResponseFields[] = [], stateOptions: ChannelStateOptions = {}, queryChannelsOptions?: ChannelOptions, + candidateIdsByCid?: Map>, ) { const { skipInitialization, offlineMode = false } = stateOptions; const channels: Channel[] = []; @@ -1446,7 +1458,7 @@ export class StreamChat extends ChatApi { channelState.messages.map(formatMessage), requestedPageSize, undefined, - { reconcile: true }, + { reconcile: true, candidateIds: candidateIdsByCid?.get(c.cid) }, ); } diff --git a/test/unit/client.test.js b/test/unit/client.test.js index fa851f4e0..4e371cccf 100644 --- a/test/unit/client.test.js +++ b/test/unit/client.test.js @@ -948,6 +948,34 @@ describe('StreamChat.queryChannels', async () => { stub.restore(); }); + it('reconciles a trailing offline hard-delete on channel-list re-hydrate (cold-boot path)', async () => { + const client = await getClientWithUser(); + const full = [ + generateMsg({ id: 'm5', created_at: '2023-11-14T12:00:05.000Z' }), + generateMsg({ id: 'm6', created_at: '2023-11-14T12:00:06.000Z' }), + generateMsg({ id: 'm7', created_at: '2023-11-14T12:00:07.000Z' }), + ]; + const stub = sinon.stub(client, 'queryChannels').resolves({ + channels: [{ ...mockChannelQueryResponse, messages: full }], + }); + + // First hydrate seeds the (cold) paginator with m5,m6,m7 — m7 is the newest / bottom-most. + const [channel] = await client.queryChannelsAndHydrate({ message_limit: 3 }); + expect(channel.messagePaginator.getItem('m7')).to.not.be.undefined; + + // While the app was closed, m7 (the last message) was hard-deleted. The next channel-list query + // returns the window WITHOUT it. m7 is above the newest returned message (m6), so only the + // pre-fetch head snapshot lets the reconcile prune it — the cold-boot path must supply it. + stub.resolves({ + channels: [{ ...mockChannelQueryResponse, messages: [full[0], full[1]] }], + }); + await client.queryChannelsAndHydrate({ message_limit: 3 }); + + expect(channel.messagePaginator.getItem('m7')).to.be.undefined; + + stub.restore(); + }); + it('seeds each queried channel paginator with its full message page', async () => { const client = await getClientWithUser(); const mockedChannelsQueryResponse = Array.from({ length: 10 }, (_, index) => From 894a7ccba6414bffcb4caaa15d0bbc0bf882df85 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 03:19:21 +0200 Subject: [PATCH 07/16] chore: add potential todos --- src/channel.ts | 6 ++++++ src/client.ts | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/channel.ts b/src/channel.ts index 82682902e..9143463f9 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1529,6 +1529,12 @@ export class Channel extends ChannelApi { // hard-delete (in the snapshot, absent from the page) from a message that arrives live // during the fetch (not in the snapshot). Captured here — the only place with the pre-await state — // so callers (channel.reload, watch) need not thread it. Empty on a cold open, so it is harmless. + // TODO(perf/cleanup): `headItems` materializes full message objects (intervalToItems) just to map + // them down to ids. A cheaper, clearer equivalent is a straight copy of the paginator index's own + // id set — expose `memberIds` on StoreBackedItemIndex (e.g. `snapshotMembers()` returning + // `new Set(this.memberIds)`) and use it here AND in client.queryChannelsAndHydrate. The broader + // scope (all intervals vs just the head) is inert: the reconcile only consults head ids, older + // island ids are never at/above-newest, and local messages are guarded by isServerConfirmedMessage. const candidateIds = messageSetToAddToIfDoesNotExist === 'latest' ? new Set(this.messagePaginator.headItems.map((message) => message.id)) diff --git a/src/client.ts b/src/client.ts index fe3b53a3d..3bdeda63e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1332,6 +1332,8 @@ export class StreamChat extends ChatApi { options?: QueryChannelsRequest, stateOptions: ChannelStateOptions = {}, ): Promise { + // TODO(perf/cleanup): prefer a `memberIds` snapshot over `headItems.map` here too — see the + // matching TODO in channel.query() for the full rationale. const candidateIdsByCid = new Map>(); for (const cid of Object.keys(this.activeChannels)) { const head = this.activeChannels[cid]?.messagePaginator?.headItems; From 3a235c28e1fd85adee741bf7c0bd408a8761e44f Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 11:49:52 +0200 Subject: [PATCH 08/16] fix: remove redndant comment --- src/channel.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index 9143463f9..6b15643ef 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1320,8 +1320,6 @@ export class Channel extends ChannelApi { this._reloading = true; try { const paginator = this.messagePaginator; - // Captured BEFORE the await: request our full loaded window (not the list's smaller page), and - // remember failed (unsent) messages so a disjoint rebuild does not silently drop them. const requestedLimit = paginator.items?.length || paginator.pageSize; const failedBefore = (paginator.items ?? []).filter( (message) => message.status === 'failed', From ad291a72cbc23cef970263b38df94c1e98475e7e Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 14:26:24 +0200 Subject: [PATCH 09/16] feat: expose batching and batch failed messages --- src/channel.ts | 11 +++++++--- src/pagination/paginators/BasePaginator.ts | 22 +++++++++++++++++++ .../paginators/MessageIntervalPaginator.ts | 16 ++++++++------ 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index 6b15643ef..0a5b71e84 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1328,9 +1328,14 @@ export class Channel extends ChannelApi { await this.watch({ messages: { limit: requestedLimit } }); this.offlineMode = false; - for (const failed of failedBefore) { - if (!paginator.getItem(failed.id)) paginator.ingestItem(failed); - } + paginator.batch( + () => { + for (const failed of failedBefore) { + if (!paginator.getItem(failed.id)) paginator.ingestItem(failed); + } + }, + { flush: true }, + ); } finally { this._reloading = false; } diff --git a/src/pagination/paginators/BasePaginator.ts b/src/pagination/paginators/BasePaginator.ts index 67118d96c..2baf4d50f 100644 --- a/src/pagination/paginators/BasePaginator.ts +++ b/src/pagination/paginators/BasePaginator.ts @@ -2264,6 +2264,28 @@ export abstract class BasePaginator { return true; } + /** + * Run `fn` as one batched mutation over this paginator's items, collapsing the redundant emits a + * naive per item loop would produce, in two independent ways: + * + * - **Shared-store fan-out** (the `_itemIndex.batch` wrapper): per-item store notifications fold + * into a single flush, so sibling holders of the same ids (e.g. a thread / pinned paginator + * sharing the entity) re-project once, not once per item. Inert for single-home paginators + * (channels, reminders, user groups) whose index is over a private store with no sibling + * subscribers — there it is a plain passthrough. + * - **This paginator's own active window** (`flush: true`): when state throttling is on (the message + * list in production), each `ingestItem` / `removeItem` inside `fn` defers its window publish via + * {@link scheduleWindowPublish}; the trailing {@link flushPendingPublishes} then emits the settled + * window exactly once. Pass it for oneshot operations (reconciliation) that must settle + * synchronously. Leave it `false` inside WS event handlers so successive events keep coalescing + * across the throttle trailing edge instead of each forcing an emit. Flushing is of course still + * possible if that is deemed necessary at a certain point. + */ + batch(fn: () => void, { flush = false }: { flush?: boolean } = {}): void { + this._itemIndex.batch(fn); + if (flush) this.flushPendingPublishes(); + } + // --------------------------------------------------------------------------- // Remove / contains // --------------------------------------------------------------------------- diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index b3f9c5d60..a2fa9eed0 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -843,10 +843,12 @@ export class MessageIntervalPaginator extends BasePaginator< */ private removeReconciledIds(ids: string[]) { if (!ids.length) return; - this._itemIndex.batch(() => { - for (const id of ids) this.removeItem({ id }); - }); - this.flushPendingPublishes(); + this.batch( + () => { + for (const id of ids) this.removeItem({ id }); + }, + { flush: true }, + ); this.purgeReconciledFromOfflineDb(ids); } @@ -1099,7 +1101,7 @@ export class MessageIntervalPaginator extends BasePaginator< // Batch: one logical operation touches many messages; coalesce the shared-store fan-out to a // single flush (sibling holders are notified once) instead of once per affected message. - this._itemIndex.batch(() => { + this.batch(() => { for (const message of loadedMessages) { if (message.user?.id === userId) { if (hardDelete) { @@ -1145,7 +1147,7 @@ export class MessageIntervalPaginator extends BasePaginator< // Batch: several cached messages may quote the updated one; coalesce the shared-store fan-out // to a single flush instead of one per re-ingested quoting message. - this._itemIndex.batch(() => { + this.batch(() => { for (const cachedMessage of cachedMessages) { if (cachedMessage.quoted_message_id !== message.id) continue; @@ -1171,7 +1173,7 @@ export class MessageIntervalPaginator extends BasePaginator< let activeAffected = false; // Batch: a user rename can touch many messages; coalesce the shared-store fan-out to sibling // holders into a single flush. This paginator's own active window is re-emitted once below. - this._itemIndex.batch(() => { + this.batch(() => { for (const message of this._itemIndex.values()) { if (message.user?.id !== user.id) continue; this._itemIndex.setOne({ ...message, user }); From a93784f5d88532e57df4959e68e88c19b544a9a5 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 14:56:08 +0200 Subject: [PATCH 10/16] chore: add test --- .../paginators/MessagePaginator.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index 7a4c373d0..5629ad071 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2324,6 +2324,49 @@ describe('MessagePaginator', () => { expect(paginator.getItem('msg-004')).toBeDefined(); }); + it('over-request: a server-capped page keeps hasMoreTail and anchors the tail cursor to the true loaded oldest', () => { + const all = Array.from({ length: 105 }, (_, i) => + msg(`msg-${String(i).padStart(3, '0')}`, i), + ); + const paginator = loadHead(all, { isTail: true }); + + paginator.mergeNewestPage(all.slice(5), { + candidateIds: new Set(all.map((message) => message.id)), + requestedLimit: 105, + }); + + const state = paginator.state.getLatestValue(); + expect(state.hasMoreTail).toBe(true); + expect(state.cursor?.tailward).toBe('msg-000'); + }); + + it('reached channel start: a page shorter than the clamped limit clears hasMoreTail, nulls the tail cursor and sets isTail', () => { + // The complementary branch: the loaded window believes older messages exist (isTail:false → + // hasMoreTail true), then the newest two are hard-deleted so the reconnect page comes back short + // of the requested limit. A short page (3 < min(5,100)) proves we reached the channel start, so + // hasMoreTail drops to false, the tail cursor nulls, and the interval's isTail flips true. + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); + + // m4 + m5 hard-deleted while offline: only the surviving newest come back. + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { + candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), + requestedLimit: 5, + }); + + const state = paginator.state.getLatestValue(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + expect(state.hasMoreTail).toBe(false); + expect(state.cursor?.tailward).toBeNull(); + expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(true); + }); + it('reconciles a deletion inside the returned page while keeping messages beyond it', () => { const all = Array.from({ length: 105 }, (_, i) => msg(`msg-${String(i).padStart(3, '0')}`, i), From f7626b731971ccc24d7e4b56daba66566447ae31 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 16:33:39 +0200 Subject: [PATCH 11/16] feat: add list level synchronous batching --- src/channel.ts | 2 +- src/pagination/paginators/BasePaginator.ts | 89 ++++++++++++++----- .../paginators/MessageIntervalPaginator.ts | 2 +- .../paginators/MessagePaginator.test.ts | 85 ++++++++++++++++++ 4 files changed, 153 insertions(+), 25 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index 0a5b71e84..d7d94bc82 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1334,7 +1334,7 @@ export class Channel extends ChannelApi { if (!paginator.getItem(failed.id)) paginator.ingestItem(failed); } }, - { flush: true }, + { coalesce: true }, ); } finally { this._reloading = false; diff --git a/src/pagination/paginators/BasePaginator.ts b/src/pagination/paginators/BasePaginator.ts index 2baf4d50f..a4c2ab984 100644 --- a/src/pagination/paginators/BasePaginator.ts +++ b/src/pagination/paginators/BasePaginator.ts @@ -443,6 +443,15 @@ export abstract class BasePaginator { /** Changed ids buffered since the last {@link flushIntervalViewPublish} (throttled paginators only). */ private _pendingViewChangedIds = new Set(); + /** + * Depth of active {@link batch} `coalesce` scopes. While > 0, `ingestItem` / `removeItem` record + * that the active window changed (see {@link _suspendedWindowDirty}) instead of publishing it, so + * the whole batch produces a single `state.items` emit — independent of state throttling. + */ + private _windowPublishSuspendDepth = 0; + /** Set by a suspended op that changed the active window, so {@link batch} publishes once on exit. */ + private _suspendedWindowDirty = false; + /** * Intervals keep items in disconnected ranges. * That is a scenario of jumping to non-sequential pages. @@ -896,6 +905,11 @@ export abstract class BasePaginator { ); } + /** True while a coalescing {@link batch} scope is suspending this paginator's own window publishes. */ + protected get isWindowPublishSuspended(): boolean { + return this._windowPublishSuspendDepth > 0; + } + /** Re-project the active window from its (live, source-of-truth) interval. `undefined` when inactive. */ private projectActiveWindow(): T[] | undefined { if (!this._activeIntervalId) return undefined; @@ -2108,7 +2122,9 @@ export abstract class BasePaginator { // 3. If it no longer matches the filter, we’re done (it has been removed above). if (!this.matchesFilter(ingestedItem)) { // Throttled: the removal above deferred its emit — publish the (settled) window once. - if (this.isStateThrottled && itemHasBeenRemoved) this.scheduleWindowPublish(); + // Suspended (coalescing batch): removeItemAtCoordinates recorded it; batch() emits once on exit. + if (!this.isWindowPublishSuspended && this.isStateThrottled && itemHasBeenRemoved) + this.scheduleWindowPublish(); return itemHasBeenRemoved; } @@ -2190,7 +2206,12 @@ export abstract class BasePaginator { // Falls somewhere *inside* the global bounds, but we don't have that page loaded. // We’ve already removed any old occurrence, so from the paginator's perspective // this item won't be visible again until the relevant page is fetched. - if (this.isStateThrottled && itemHasBeenRemoved) this.scheduleWindowPublish(); + if ( + !this.isWindowPublishSuspended && + this.isStateThrottled && + itemHasBeenRemoved + ) + this.scheduleWindowPublish(); return itemHasBeenRemoved; } } @@ -2211,7 +2232,10 @@ export abstract class BasePaginator { activeIntervalIdBeforeRemoval === removedIntervalId && targetInterval.id === removedIntervalId ) { - this.setActiveInterval(targetInterval); + this.setActiveInterval( + targetInterval, + this.isWindowPublishSuspended ? { updateState: false } : undefined, + ); } const addedNewInterval = !this._itemIntervals.has(targetInterval.id); @@ -2228,7 +2252,10 @@ export abstract class BasePaginator { this._activeIntervalId, ) ) { - if (this.isStateThrottled) { + if (this.isWindowPublishSuspended) { + // Coalescing batch: record the change; batch() emits the settled window once on exit. + this._suspendedWindowDirty = true; + } else if (this.isStateThrottled) { this.scheduleWindowPublish(); } else { const items = this.items ?? []; @@ -2266,24 +2293,37 @@ export abstract class BasePaginator { /** * Run `fn` as one batched mutation over this paginator's items, collapsing the redundant emits a - * naive per item loop would produce, in two independent ways: + * naive per-item loop would produce: * - * - **Shared-store fan-out** (the `_itemIndex.batch` wrapper): per-item store notifications fold - * into a single flush, so sibling holders of the same ids (e.g. a thread / pinned paginator - * sharing the entity) re-project once, not once per item. Inert for single-home paginators - * (channels, reminders, user groups) whose index is over a private store with no sibling - * subscribers — there it is a plain passthrough. - * - **This paginator's own active window** (`flush: true`): when state throttling is on (the message - * list in production), each `ingestItem` / `removeItem` inside `fn` defers its window publish via - * {@link scheduleWindowPublish}; the trailing {@link flushPendingPublishes} then emits the settled - * window exactly once. Pass it for oneshot operations (reconciliation) that must settle - * synchronously. Leave it `false` inside WS event handlers so successive events keep coalescing - * across the throttle trailing edge instead of each forcing an emit. Flushing is of course still - * possible if that is deemed necessary at a certain point. - */ - batch(fn: () => void, { flush = false }: { flush?: boolean } = {}): void { - this._itemIndex.batch(fn); - if (flush) this.flushPendingPublishes(); + * - **Shared-store fan-out** (always, via the `_itemIndex.batch` wrapper): per-item store + * notifications fold into a single flush, so sibling holders of the same ids (e.g. a thread / + * pinned paginator sharing the entity) re-project once, not once per item. Inert for single-home + * paginators (channels, reminders, user groups) whose index is over a private store with no + * sibling subscribers — there it is a plain passthrough. + * - **This paginator's own active window** (`coalesce: true`): every `ingestItem` / `removeItem` + * inside `fn` records that the window changed instead of publishing it, then `batch` emits the + * settled window exactly once on exit via {@link flushWindowPublish}. This is deterministic and + * independent of state throttling — unlike leaving it off, where an un-throttled paginator emits + * once per item and a throttled one merely coalesces within its 500ms window. Use it for one-shot + * operations (reconciliation, reload re-ingest) that must settle in a single update. Omit it in WS + * event handlers, where per-event publishes should ride the throttle so successive events coalesce + * across its trailing edge rather than each forcing a synchronous emit. + */ + batch(fn: () => void, { coalesce = false }: { coalesce?: boolean } = {}): void { + if (!coalesce) { + this._itemIndex.batch(fn); + return; + } + this._windowPublishSuspendDepth += 1; + try { + this._itemIndex.batch(fn); + } finally { + this._windowPublishSuspendDepth -= 1; + } + if (this._windowPublishSuspendDepth === 0 && this._suspendedWindowDirty) { + this._suspendedWindowDirty = false; + this.flushWindowPublish(); + } } // --------------------------------------------------------------------------- @@ -2317,7 +2357,9 @@ export abstract class BasePaginator { // 2) Remove from visible state.items, if present if (stateLocation && stateLocation.currentIndex > -1) { - if (!this.isStateThrottled) { + if (this.isWindowPublishSuspended) { + this._suspendedWindowDirty = true; + } else if (!this.isStateThrottled) { const newItems = [...(this.items ?? [])]; newItems.splice(stateLocation.currentIndex, 1); this.state.partialNext({ items: newItems }); @@ -2355,7 +2397,8 @@ export abstract class BasePaginator { const result = this.removeItemAtCoordinates(coords); this._itemIndex.remove(this.getItemId(item)); // Throttled: removeItemAtCoordinates deferred its emit — publish the (settled) window once. - if (this.isStateThrottled) this.scheduleWindowPublish(); + if (!this.isWindowPublishSuspended && this.isStateThrottled) + this.scheduleWindowPublish(); return result; } diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index a2fa9eed0..edd3f4e7c 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -847,7 +847,7 @@ export class MessageIntervalPaginator extends BasePaginator< () => { for (const id of ids) this.removeItem({ id }); }, - { flush: true }, + { coalesce: true }, ); this.purgeReconciledFromOfflineDb(ids); } diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index 5629ad071..b2df3fb40 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2367,6 +2367,91 @@ describe('MessagePaginator', () => { expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(true); }); + describe('batch({ coalesce: true }) — single deterministic window publish', () => { + it('coalesces N removals into a single state publish', () => { + const paginator = loadHead([ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + ]); + const spy = vi.spyOn(paginator.state, 'partialNext'); + + paginator.batch( + () => { + paginator.removeItem({ id: 'm1' }); + paginator.removeItem({ id: 'm2' }); + paginator.removeItem({ id: 'm3' }); + }, + { coalesce: true }, + ); + + expect(spy).toHaveBeenCalledTimes(1); + expect(ids(paginator)).toEqual(['m4']); + }); + + it('coalesces N in-place updates into a single state publish', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + const spy = vi.spyOn(paginator.state, 'partialNext'); + + paginator.batch( + () => { + paginator.ingestItem(msg('m1', 1, { text: 'a' })); + paginator.ingestItem(msg('m2', 2, { text: 'b' })); + paginator.ingestItem(msg('m3', 3, { text: 'c' })); + }, + { coalesce: true }, + ); + + expect(spy).toHaveBeenCalledTimes(1); + expect(paginator.getItem('m1')?.text).toBe('a'); + expect(paginator.getItem('m3')?.text).toBe('c'); + }); + + it('coalesces a mixed remove + ingest batch into a single state publish', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + const spy = vi.spyOn(paginator.state, 'partialNext'); + + paginator.batch( + () => { + paginator.removeItem({ id: 'm2' }); + paginator.ingestItem(msg('m3', 3, { text: 'edited' })); + }, + { coalesce: true }, + ); + + expect(spy).toHaveBeenCalledTimes(1); + expect(ids(paginator)).toEqual(['m1', 'm3']); + expect(paginator.getItem('m3')?.text).toBe('edited'); + }); + + it('without coalesce, the same removals publish once per item (proves the scope does the work)', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + const spy = vi.spyOn(paginator.state, 'partialNext'); + + paginator.batch(() => { + paginator.removeItem({ id: 'm1' }); + paginator.removeItem({ id: 'm2' }); + }); + + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('does not publish when the coalesced batch leaves the active window unchanged', () => { + const paginator = loadHead([msg('m1', 1), msg('m2', 2)]); + const spy = vi.spyOn(paginator.state, 'partialNext'); + + paginator.batch( + () => { + paginator.removeItem({ id: 'does-not-exist' }); + }, + { coalesce: true }, + ); + + expect(spy).not.toHaveBeenCalled(); + }); + }); + it('reconciles a deletion inside the returned page while keeping messages beyond it', () => { const all = Array.from({ length: 105 }, (_, i) => msg(`msg-${String(i).padStart(3, '0')}`, i), From 05953603678ef5da31f2dfa5a63eff9543712ecb Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 16:37:15 +0200 Subject: [PATCH 12/16] fix: rename --- .../paginators/MessageIntervalPaginator.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index edd3f4e7c..cdec7719b 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -615,7 +615,7 @@ export class MessageIntervalPaginator extends BasePaginator< * partial page leaves it untouched, while a page reaching deeper than the loaded window (or a * stale offline-DB cursor) re-anchors it to the true loaded oldest so "load older" keeps working — * derived from the interval, never the page's length. Then destructive reconciliation - * ({@link reconcileLoadedAgainstPage}) removes any loaded message + * ({@link reconcileHeadAgainstPage}) removes any loaded message * that the authoritative page proves was hard-deleted while offline (see that method + the * {@link MergeNewestPageOptions} for the exact, safe window). * @@ -647,7 +647,7 @@ export class MessageIntervalPaginator extends BasePaginator< // targets the head interval regardless of which interval is active, and removing a hidden-head ghost // leaves the active island's items untouched. if (!this.isActiveInterval(headInterval)) { - this.reconcileLoadedAgainstPage(page, options); + this.reconcileHeadAgainstPage(page, options); return; } @@ -655,7 +655,7 @@ export class MessageIntervalPaginator extends BasePaginator< // Empty page: never blanks the list by itself. Only when the caller supplied a pre-fetch // snapshot do we treat it as authoritative "channel emptied" and remove ghosts (a message that // arrived live during the fetch is excluded by the snapshot). - this.reconcileLoadedAgainstPage([], options); + this.reconcileHeadAgainstPage([], options); return; } @@ -731,7 +731,7 @@ export class MessageIntervalPaginator extends BasePaginator< }); // With the newest page merged in, drop any loaded message the page proves was hard-deleted. - this.reconcileLoadedAgainstPage(page, options); + this.reconcileHeadAgainstPage(page, options); }; /** @@ -749,9 +749,9 @@ export class MessageIntervalPaginator extends BasePaginator< } /** - * Destructive half of {@link mergeNewestPage}: remove loaded messages that the freshly-fetched - * newest `page` proves were hard-deleted while offline (a hard delete emits no event to other - * clients, and the merge is additive, so they would otherwise linger forever). + * Destructive half of {@link mergeNewestPage}: remove messages in the loaded head interval that the + * freshly-fetched newest `page` proves were hard-deleted while offline (a hard delete emits no event + * to other clients, and the merge is additive, so they would otherwise linger forever). * * The reconcilable window is derived ENTIRELY from what the page returned — never a hardcoded page * size — so it can only ever remove messages the page actually covers: @@ -771,7 +771,7 @@ export class MessageIntervalPaginator extends BasePaginator< * store and — via the {@link MessagePaginator} override — the tracked last message all stay * correct, then the active window is re-emitted once. */ - protected reconcileLoadedAgainstPage( + protected reconcileHeadAgainstPage( page: LocalMessage[], options?: MergeNewestPageOptions, ) { From fa09af4f5a04835fcb2aed7d45f6d2212c9f74b4 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 17:04:24 +0200 Subject: [PATCH 13/16] chore: rename api --- .../paginators/MessageIntervalPaginator.ts | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index cdec7719b..85916ef5a 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -790,7 +790,7 @@ export class MessageIntervalPaginator extends BasePaginator< const message = this.getItem(id); return !!message && this.isServerConfirmedMessage(message); }); - this.removeReconciledIds(toRemove); + this.removeReconciledItems(toRemove); return; } @@ -831,17 +831,28 @@ export class MessageIntervalPaginator extends BasePaginator< if (ts > windowLowTs) toRemove.push(id); } - this.removeReconciledIds(toRemove); + this.removeReconciledItems(toRemove); } /** - * Remove a set of reconciled (hard-deleted) ids in one batch — coalescing the shared-store fan-out - * to a single flush — then flush the deferred window publish so the list drops the ghosts - * synchronously (blanking to `[]` if the active window emptied, per {@link flushWindowPublish}). - * Finally, mirror the removal into the offline DB so a cold start does not re-seed the ghosts from - * SQLite. No-op for an empty set, so an unaffected merge does not touch state a second time. + * Remove a set of reconciled (hard-deleted) ids from this paginator's loaded state in one coalesced + * batch. Each id goes through {@link removeItem}, which drops it from THREE places — not a single + * interval: + * + * 1. the item index — unlinks this paginator's membership from the shared entity store (GC'ing the + * content if this was the last holder); + * 2. the interval that holds it — located by id, so it is NOT head-specific in general; in practice + * it is always the head interval, because the sole caller ({@link reconcileHeadAgainstPage}) + * sources these ids from `headInterval.itemIds` and an item has single-interval membership; + * 3. the active window (`state.items`) if visible — blanking to `[]` when the window empties, per + * {@link flushWindowPublish}. + * + * The coalesced batch collapses all of that into a single `state.items` publish. Then mirror the + * removal into the offline DB (see {@link purgeReconciledFromOfflineDb}) so a cold start does not + * re-seed the ghosts from SQLite. No-op for an empty set, so an unaffected merge does not touch + * state a second time. */ - private removeReconciledIds(ids: string[]) { + private removeReconciledItems(ids: string[]) { if (!ids.length) return; this.batch( () => { From 269f4d6ea828d75cdc3b6f82c8d92a1004a84d0d Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 12 Aug 2026 17:08:32 +0200 Subject: [PATCH 14/16] fix: clarify docs --- .../paginators/MessageIntervalPaginator.ts | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index 85916ef5a..6870a5974 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -148,27 +148,29 @@ export type MessagePaginatorOptions = { * offline. A hard delete emits no event to other clients, and the merge is otherwise additive, so * without this such a message lingers as a ghost after reconnect. * - * With NO options, `mergeNewestPage` still prunes any loaded message that falls WITHIN the returned - * page's `created_at` span but is absent from it — unconditionally safe (a message that arrived live - * during the caller's fetch is always strictly newer than the newest returned message, so it can - * never fall in that span). The options widen the reconcilable window: + * Throughout, `page` is the newest window the server returned for `mergeNewestPage`, and "`page`'s + * newest / oldest message" are its bounds by `created_at`. With NO options, `mergeNewestPage` still + * prunes any loaded message that falls WITHIN `page`'s `created_at` span but is absent from it — + * unconditionally safe (a message that arrived live during the caller's fetch is always strictly + * newer than `page`'s newest message, so it can never fall in that span). The options widen the + * reconcilable window: */ export type MergeNewestPageOptions = { /** - * The `limit` the caller passed to the query that produced the page. Lets reconciliation tell - * "the page reached the channel's oldest message" (a returned count short of the request) from - * "the page is full and older messages remain". Only then may it prune loaded messages OLDER than - * the oldest returned message (e.g. the oldest loaded message was the one deleted). Clamped to the + * The `limit` the caller passed to the query that produced `page`. Lets reconciliation tell "`page` + * reached the channel's own oldest message" (it came back shorter than requested) from "`page` is + * full and older messages remain". Only in the former may it prune loaded messages OLDER than + * `page`'s oldest message (e.g. the oldest loaded message was the one deleted). Clamped to the * server's max page size so an over-request cannot be mistaken for reaching the start. */ requestedLimit?: number; /** * A snapshot of the loaded message ids taken BEFORE the caller's fetch await. Required to prune - * messages NEWER than the newest returned message (a hard-deleted newest message) and to reconcile - * an empty page (a fully-emptied channel): at/above that top edge a just-deleted message and a - * message that arrived live during the fetch are indistinguishable by timestamp — only the - * pre-fetch snapshot separates them (a live arrival is not in it). Must be captured before the - * await; the paginator's own items at merge time already include any live arrival. + * messages NEWER than `page`'s newest message (a hard-deleted newest message) and to reconcile an + * empty `page` (a fully-emptied channel): at/above that top edge a just-deleted message and a + * message that arrived live during the fetch are indistinguishable by timestamp — only the pre-fetch + * snapshot separates them (a live arrival is not in it). Must be captured before the await; the + * paginator's own items at merge time already include any live arrival. */ candidateIds?: ReadonlySet; }; @@ -749,27 +751,31 @@ export class MessageIntervalPaginator extends BasePaginator< } /** - * Destructive half of {@link mergeNewestPage}: remove messages in the loaded head interval that the - * freshly-fetched newest `page` proves were hard-deleted while offline (a hard delete emits no event - * to other clients, and the merge is additive, so they would otherwise linger forever). + * Destructive half of {@link mergeNewestPage}: remove messages in the loaded head interval that + * `page` proves were hard-deleted while offline (a hard delete emits no event to other clients, and + * the merge is additive, so they would otherwise linger forever). * - * The reconcilable window is derived ENTIRELY from what the page returned — never a hardcoded page - * size — so it can only ever remove messages the page actually covers: + * Here `page` is the freshly-fetched newest window the server returned for the query that produced + * it. Its two bounds — used throughout below — are its NEWEST message (`page`'s last item by + * `created_at`) and its OLDEST message (`page`'s first item). The reconcilable window is derived + * ENTIRELY from those two bounds — never a hardcoded page size — so it can only ever remove messages + * `page` actually covers. Three regions, by a loaded message's `created_at`: * - * - WITHIN the page's span (`oldest returned < created_at < newest returned`): a server-confirmed - * loaded message absent from the page was hard-deleted. Safe with no snapshot — a message that - * arrived live during the caller's fetch is always strictly newer than the newest returned - * message, so it can never fall in this span. - * - BELOW the oldest returned message: only reconcilable when the page reached the channel's oldest - * message (`requestedLimit` given and the page came back short, clamped to the server max page - * size). Otherwise older messages simply were not fetched and are left untouched. - * - AT/ABOVE the newest returned message (a hard-deleted newest message) and the empty-page case: - * only reconcilable with a pre-fetch `candidateIds` snapshot, which alone distinguishes a ghost - * from a live arrival at that top edge. + * - WITHIN `page` (strictly between `page`'s oldest and newest message): a server-confirmed loaded + * message absent from `page` was hard-deleted. Safe with no snapshot — a message that arrived live + * during the caller's fetch is always strictly newer than `page`'s newest message, so it can never + * fall in this span. + * - BELOW `page`'s oldest message: only reconcilable when `page` reached the channel's own oldest + * message (`requestedLimit` given and `page` came back shorter than requested, clamped to the + * server's max page size). Otherwise older messages simply were not fetched, so they are left + * untouched. + * - AT/ABOVE `page`'s newest message (a hard-deleted newest message), and the empty-`page` case: + * only reconcilable with a pre-fetch `candidateIds` snapshot, which alone tells a just-deleted + * message from one that arrived live during the fetch at that top edge. * * Removal goes through {@link removeItem} (batched) so the item index, intervals, shared message - * store and — via the {@link MessagePaginator} override — the tracked last message all stay - * correct, then the active window is re-emitted once. + * store and — via the {@link MessagePaginator} override — the tracked last message all stay correct; + * the active window is then re-emitted once. */ protected reconcileHeadAgainstPage( page: LocalMessage[], From 679e8e417d9af06435e56474a800d3bb09c08400 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Thu, 13 Aug 2026 14:45:40 +0200 Subject: [PATCH 15/16] fix: properly reconcile above the fold items --- src/channel.ts | 7 +- .../paginators/MessageIntervalPaginator.ts | 207 +++++++--- .../paginators/MessagePaginator.test.ts | 353 +++++++++++++++--- 3 files changed, 447 insertions(+), 120 deletions(-) diff --git a/src/channel.ts b/src/channel.ts index d7d94bc82..65f3485cd 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -1320,10 +1320,9 @@ export class Channel extends ChannelApi { this._reloading = true; try { const paginator = this.messagePaginator; - const requestedLimit = paginator.items?.length || paginator.pageSize; - const failedBefore = (paginator.items ?? []).filter( - (message) => message.status === 'failed', - ); + const headItems = paginator.headItems; + const requestedLimit = headItems.length || paginator.pageSize; + const failedBefore = headItems.filter((message) => message.status === 'failed'); await this.watch({ messages: { limit: requestedLimit } }); this.offlineMode = false; diff --git a/src/pagination/paginators/MessageIntervalPaginator.ts b/src/pagination/paginators/MessageIntervalPaginator.ts index 6870a5974..1121d868d 100644 --- a/src/pagination/paginators/MessageIntervalPaginator.ts +++ b/src/pagination/paginators/MessageIntervalPaginator.ts @@ -149,19 +149,24 @@ export type MessagePaginatorOptions = { * without this such a message lingers as a ghost after reconnect. * * Throughout, `page` is the newest window the server returned for `mergeNewestPage`, and "`page`'s - * newest / oldest message" are its bounds by `created_at`. With NO options, `mergeNewestPage` still - * prunes any loaded message that falls WITHIN `page`'s `created_at` span but is absent from it — + * newest / oldest message" are its bounds by `created_at`. With NO options, `mergeNewestPage` prunes + * any loaded message that falls WITHIN `page`'s `created_at` span but is absent from it — * unconditionally safe (a message that arrived live during the caller's fetch is always strictly - * newer than `page`'s newest message, so it can never fall in that span). The options widen the - * reconcilable window: + * newer than `page`'s newest message, so it can never fall in that span). `candidateIds` widens the + * reconcilable window at the TOP edge; `requestedLimit` only tunes the `hasMoreTail` affordance and + * can never remove a message: */ export type MergeNewestPageOptions = { /** - * The `limit` the caller passed to the query that produced `page`. Lets reconciliation tell "`page` - * reached the channel's own oldest message" (it came back shorter than requested) from "`page` is - * full and older messages remain". Only in the former may it prune loaded messages OLDER than - * `page`'s oldest message (e.g. the oldest loaded message was the one deleted). Clamped to the - * server's max page size so an over-request cannot be mistaken for reaching the start. + * The `limit` the caller passed to the query that produced `page`. Used ONLY to derive + * `hasMoreTail` — whether `page` reached the channel's own oldest message — by comparing it to + * `page.length`. It NEVER removes a message (reconciliation stays window-only regardless), so the + * worst a wrong value can do is show/hide the "load older" affordance for one settle. Trusted only + * when no larger than the paginator's own `pageSize` (see + * {@link MessageIntervalPaginator.pageReachedChannelStart}); a larger over-request — e.g. + * `channel.reload` re-fetching the whole loaded window to reconcile as much as possible — may have + * been silently server-capped, so its short page is ignored rather than mistaken for reaching the + * start. */ requestedLimit?: number; /** @@ -215,6 +220,8 @@ export class MessageIntervalPaginator extends BasePaginator< protected _requestSort = DEFAULT_BACKEND_SORT; protected _itemOrder: MessagePaginatorSort = DEFAULT_BACKEND_SORT; protected _nextQueryShape: MessageQueryShape | undefined; + /** Pending below-window reached-start probe (exposed for deterministic test awaiting). */ + private _belowWindowReconcile?: Promise; sortComparator: (a: LocalMessage, b: LocalMessage) => number; /** * Single source of truth for whether a message should be included in paginator intervals/state. @@ -613,10 +620,10 @@ export class MessageIntervalPaginator extends BasePaginator< * 1. OVERLAP - the incoming page shares at least one id with the loaded head (fewer than a full * page is new). Merge in place: existing items are reconciled by id (edits, soft deletes), new * items are appended and every already loaded item (including older pages already paged in) is - * kept. The tail boundary (`hasMoreTail`/`cursor.tailward`) is taken from the MERGED interval: a - * partial page leaves it untouched, while a page reaching deeper than the loaded window (or a - * stale offline-DB cursor) re-anchors it to the true loaded oldest so "load older" keeps working — - * derived from the interval, never the page's length. Then destructive reconciliation + * kept. The tail boundary (`hasMoreTail`/`cursor.tailward`) is set from whether the page reached + * the channel start ({@link pageReachedChannelStart}): a bounded page that came back short means + * no more older, otherwise "load older" stays enabled and the cursor re-anchors to the true loaded + * oldest (which also clears a stale offline-DB cursor). Then destructive reconciliation * ({@link reconcileHeadAgainstPage}) removes any loaded message * that the authoritative page proves was hard-deleted while offline (see that method + the * {@link MergeNewestPageOptions} for the exact, safe window). @@ -642,6 +649,10 @@ export class MessageIntervalPaginator extends BasePaginator< mergeNewestPage = (page: LocalMessage[], options?: MergeNewestPageOptions) => { const headInterval = this.itemIntervals[0] as Interval | undefined; if (!headInterval?.isHead) return; + // Captured BEFORE the merge overwrites state — inputs for the below-window reached-start probe + // (overlap branch only): did we believe we were at the channel start, and how much was loaded. + const wasAtChannelStart = !this.state.getLatestValue().hasMoreTail; + const preMergeLoadedCount = headInterval.itemIds.length; // If the caller jumped to a separate (older) window, that window is active and the head is merely // still-loaded underneath. Don't MERGE the fresh page or switch the view (that would yank them to // the newest) — but STILL prune offline hard-deletes out of the hidden head, otherwise returning to @@ -694,29 +705,17 @@ export class MessageIntervalPaginator extends BasePaginator< const interval = this.ingestPage({ page, isHead: true, setActive: false }); if (!interval) return; - // Re-compute hasMoreTail from the FETCHED PAGE, not the merged interval. The interval's isTail is - // "sticky" — mergeTwoAnchoredIntervals ORs isTail — so a stale offline-DB window persisted as - // "complete" (isTail:true) keeps hasMoreTail=false through the merge, and "load older" stays dead. - // A full page (length == the limit we asked for) means older messages remain; a short page means we - // reached the channel start. Without a caller-supplied limit (a live-edit merge of unknown size) - // fall back to the interval's own flag. Pagination reads off STATE, so writing it here is what - // unblocks "load older"; a later executeQuery re-derives per page. Mirrors postQueryReconcile, - // which likewise derives this flag from the page rather than the interval. - const { requestedLimit } = options ?? {}; - const canDeriveTail = typeof requestedLimit === 'number'; - const reachedChannelStart = - canDeriveTail && - page.length < Math.min(requestedLimit, DEFAULT_CHANNEL_MESSAGE_LIST_PAGE_SIZE); - const hasMoreTail = canDeriveTail ? !reachedChannelStart : interval.hasMoreTail; - - // Correct the INTERVAL flags too, not just state. `intervalsOverlap` (the merge test in - // ingestPage) consults `interval.isTail`: a sticky `isTail:true` inherited from a "complete" - // offline-DB window makes ANY older page count as overlapping this head interval — so a far - // jump's load-older welds two pages-apart sets into one. Derive isTail from the page here like the - // state above; mirrors postQueryReconcile (`interval.isTail = hasMoreTail === false`). Only a - // genuine channel-start interval (no older messages) keeps isTail:true. + // hasMoreTail: does the fetched page prove it reached the channel's own oldest message? Only a + // request bounded by our OWN pageSize gives that proof (a short page); an over-request may have + // been server-capped, so it biases to `true` — never asserting "no more older" from a page that + // may be truncated. See {@link pageReachedChannelStart} (no hardcoded max page size, per-paginator). + // This decides only the "load older" AFFORDANCE — it never removes a message (reconciliation stays + // window-only below). isTail mirrors it: `true` only when we RELIABLY reached the start, which also + // clears a stale offline-DB `isTail:true` and stops `intervalsOverlap` welding a far page across a + // gap. The tail cursor anchors to the oldest loaded id so "load older" stays contiguous. + const hasMoreTail = !this.pageReachedChannelStart(page, options); interval.hasMoreTail = hasMoreTail; - interval.isTail = hasMoreTail === false; + interval.isTail = !hasMoreTail; this.setActiveInterval(interval, { updateState: false }); this.state.partialNext({ @@ -732,10 +731,92 @@ export class MessageIntervalPaginator extends BasePaginator< }, }); - // With the newest page merged in, drop any loaded message the page proves was hard-deleted. - this.reconcileHeadAgainstPage(page, options); + // With the newest page merged in, drop any loaded message the page proves was hard-deleted, and + // collect the below-window leftovers it cannot decide from the page alone. + const belowWindow = this.reconcileHeadAgainstPage(page, options); + + // The leftovers are ambiguous (hard-deleted oldest vs server-capped over-request). Only a + // full-window reload (`requestedLimit >= loaded`, so NOT the small list hydrate) that we believed + // reached the start could have reached it — probe to settle them there; anything else can't, so skip. + const requestedLimit = options?.requestedLimit; + if ( + belowWindow.length && + wasAtChannelStart && + typeof requestedLimit === 'number' && + requestedLimit >= preMergeLoadedCount + ) { + this._belowWindowReconcile = this.pruneBelowWindowIfReachedStart( + belowWindow, + this.getItemId(page[0]), + ); + } }; + /** + * Whether `page` proves it reached the channel's own oldest message: the caller's request was + * bounded by this paginator's OWN `pageSize` AND `page` came back shorter than that request. + * + * `pageSize <= the server's max page size` is already the invariant normal pagination + * (`executeQuery`) relies on — `page.length < pageSize` ⟹ reached the end — so a request no larger + * than `pageSize` returns exactly what was asked unless the channel ended. This needs no hardcoded + * max page size and is per-paginator, so a thread reply paginator uses its own `pageSize`. + * + * An OVER-request (larger than `pageSize`, e.g. `channel.reload` re-fetching the whole loaded window + * to reconcile as much as possible) may have been silently capped by the server, so a short page + * there is NOT proof of reaching the start — return `false` and stay conservative. This only ever + * gates the `hasMoreTail` affordance; it never removes a message. + */ + private pageReachedChannelStart( + page: LocalMessage[], + options?: MergeNewestPageOptions, + ): boolean { + const requestedLimit = options?.requestedLimit; + return ( + typeof requestedLimit === 'number' && + requestedLimit <= this.pageSize && + page.length < requestedLimit + ); + } + + /** + * Settle the below-window leftovers {@link reconcileHeadAgainstPage} handed back (loaded messages + * older than `anchorId` — the returned page's oldest — and absent from the page). From the page alone + * they're ambiguous: a hard-deleted oldest vs a server-capped over-request. Ask the server "anything + * older than `anchorId`?" — the only cap-free way to tell. Nothing older ⇒ genuine deletes ⇒ remove + * them and settle the tail. A probe failure or a head change across the await keeps them (never a false + * delete). The caller decides WHEN this runs. + */ + private async pruneBelowWindowIfReachedStart( + belowWindow: string[], + anchorId: string, + ): Promise { + const headId = (this.itemIntervals[0] as Interval | undefined)?.id; + // older exists (or the probe failed), we we're not provably the start, so + // keep the leftovers + if (await this.hasMessagesOlderThan(anchorId).catch(() => true)) return; + // revalidate the head across the await (a jump/reset/newer merge aborts), then remove the leftovers + const head = this.itemIntervals[0] as Interval | undefined; + if (!head?.isHead || head.id !== headId) return; + this.removeReconciledItems(belowWindow); + head.hasMoreTail = false; // we now KNOW we reached the start, so settle the "load older" affordance + head.isTail = true; + if (this.isActiveInterval(head)) { + this.state.partialNext({ + hasMoreTail: false, + cursor: { headward: null, tailward: null }, + }); + } + } + + /** One `limit: 1` fetch of messages older than `id` (channel main list or thread replies). */ + private async hasMessagesOlderThan(id: string): Promise { + const pagination = { limit: 1, id_lt: id } as MessagePaginationParams; + const { messages } = this.parentMessageId + ? await this.channel.getReplies({ parent_id: this.parentMessageId, ...pagination }) + : await this.channel.query({ messages: pagination }); + return Array.isArray(messages) && messages.length > 0; + } + /** * Whether a loaded message is server-confirmed and therefore eligible to be reconciled away when * absent from an authoritative page. Excludes local-only messages the server has never @@ -756,33 +837,38 @@ export class MessageIntervalPaginator extends BasePaginator< * the merge is additive, so they would otherwise linger forever). * * Here `page` is the freshly-fetched newest window the server returned for the query that produced - * it. Its two bounds — used throughout below — are its NEWEST message (`page`'s last item by - * `created_at`) and its OLDEST message (`page`'s first item). The reconcilable window is derived - * ENTIRELY from those two bounds — never a hardcoded page size — so it can only ever remove messages - * `page` actually covers. Three regions, by a loaded message's `created_at`: + * it. Its two bounds are its NEWEST message (`page`'s last item by `created_at`) and its OLDEST + * message (`page`'s first item). Reconciliation removes ONLY messages `page` actually covers — no + * hardcoded page size is involved — in two regions, by a loaded message's `created_at`: * * - WITHIN `page` (strictly between `page`'s oldest and newest message): a server-confirmed loaded * message absent from `page` was hard-deleted. Safe with no snapshot — a message that arrived live * during the caller's fetch is always strictly newer than `page`'s newest message, so it can never * fall in this span. - * - BELOW `page`'s oldest message: only reconcilable when `page` reached the channel's own oldest - * message (`requestedLimit` given and `page` came back shorter than requested, clamped to the - * server's max page size). Otherwise older messages simply were not fetched, so they are left - * untouched. * - AT/ABOVE `page`'s newest message (a hard-deleted newest message), and the empty-`page` case: * only reconcilable with a pre-fetch `candidateIds` snapshot, which alone tells a just-deleted * message from one that arrived live during the fetch at that top edge. * + * A loaded message BELOW `page`'s oldest is NOT removed here (window-only) — from the page alone a + * hard-deleted oldest is indistinguishable from a server-capped over-request. Those messages are + * instead RETURNED (the below-window leftovers) so {@link pruneBelowWindowIfReachedStart} can settle + * them with a `limit: 1` "anything older?" probe — the only cap-free proof — and remove them only if + * the server confirms nothing older exists. Absent that proof they are kept (a later cold/fresh query + * omits any real delete). + * * Removal goes through {@link removeItem} (batched) so the item index, intervals, shared message * store and — via the {@link MessagePaginator} override — the tracked last message all stay correct; * the active window is then re-emitted once. + * + * @returns the below-window leftover ids (loaded, server-confirmed, older than `page`'s oldest, absent + * from it) for the caller's reached-start probe. Empty for the no-head / empty-page paths. */ protected reconcileHeadAgainstPage( page: LocalMessage[], options?: MergeNewestPageOptions, - ) { + ): string[] { const headInterval = this.itemIntervals[0] as Interval | undefined; - if (!headInterval?.isHead) return; + if (!headInterval?.isHead) return []; const loadedIds = headInterval.itemIds; const candidateIds = options?.candidateIds; @@ -790,33 +876,27 @@ export class MessageIntervalPaginator extends BasePaginator< // Empty page → the channel has no messages. Every server-confirmed loaded message is gone, but // only remove ids from the pre-fetch snapshot so a message that landed during the fetch survives. if (!page.length) { - if (!candidateIds) return; + if (!candidateIds) return []; const toRemove = loadedIds.filter((id) => { if (!candidateIds.has(id)) return false; const message = this.getItem(id); return !!message && this.isServerConfirmedMessage(message); }); this.removeReconciledItems(toRemove); - return; + return []; } const pageIds = new Set(page.map((message) => this.getItemId(message))); const newestReturnedTs = getMessageCreatedAtTimestamp(page[page.length - 1]); const oldestReturnedTs = getMessageCreatedAtTimestamp(page[0]); - // Only extend below the oldest returned message when the page proves it reached the channel's - // oldest message: it came back shorter than requested. Clamp the request to the server's max page - // size so asking for MORE than one page can return (an over-request) is not mistaken for reaching - // the start — in that case the shortfall is the server capping, not the channel ending. - const { requestedLimit } = options ?? {}; - const reachedChannelStart = - typeof requestedLimit === 'number' && - page.length < Math.min(requestedLimit, DEFAULT_CHANNEL_MESSAGE_LIST_PAGE_SIZE); - const windowLowTs = reachedChannelStart - ? Number.NEGATIVE_INFINITY - : (oldestReturnedTs ?? Number.POSITIVE_INFINITY); + // Window-only: never remove below the returned page's oldest here, because from the page alone a + // hard-deleted oldest is indistinguishable from a server-capped over-request. Below-window messages + // are collected into `belowWindow` and returned instead, for the reached-start probe to settle. + const windowLowTs = oldestReturnedTs ?? Number.POSITIVE_INFINITY; const toRemove: string[] = []; + const belowWindow: string[] = []; for (const id of loadedIds) { if (pageIds.has(id)) continue; // present on the server → keep const message = this.getItem(id); @@ -832,12 +912,15 @@ export class MessageIntervalPaginator extends BasePaginator< if (candidateIds?.has(id)) toRemove.push(id); continue; } - // Strictly within the page's span (below the newest returned message): a live arrival can never - // be here, so the absence is a hard delete regardless of a snapshot. + // Below the newest returned message: within the page's span (a live arrival can never be here, so + // absence = a hard delete) → remove; at/below the page's oldest → a below-window leftover the page + // cannot decide, returned for the reached-start probe. if (ts > windowLowTs) toRemove.push(id); + else belowWindow.push(id); } this.removeReconciledItems(toRemove); + return belowWindow; } /** diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index b2df3fb40..5c2b94798 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -1736,11 +1736,10 @@ describe('MessagePaginator', () => { }); it('keeps hasMoreTail and anchors the tail cursor to the loaded oldest when merging a partial newest window', () => { - // Only the newest window is loaded and older items still exist (hasMoreTail true). Merging a - // short page (fewer than pageSize) whose first item is the set's first item must NOT clear - // hasMoreTail — re-deriving it from the page's LENGTH would wrongly break "load older". The tail - // boundary is taken from the MERGED interval, so hasMoreTail stays true and the cursor anchors to - // the loaded oldest (m1) — the correct "load older" anchor. + // Only the newest window is loaded and older items still exist (hasMoreTail true). A live partial + // merge passes no requestedLimit, so the flag stays conservative (true) — hasMoreTail is only ever + // lowered by a caller-supplied requestedLimit bounded by pageSize that comes back short. Here it + // stays true and the cursor anchors to the loaded oldest (m1) — the correct "load older" anchor. const { paginator, m1, m2 } = setupLoadedHead({ isTail: false }); expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); @@ -2018,28 +2017,29 @@ describe('MessagePaginator', () => { // m2 slid into the window: page = [m2,m3,m5,m6]. Older m1 is below the page and MUST stay. paginator.mergeNewestPage( [msg('m2', 2), msg('m3', 3), msg('m5', 5), msg('m6', 6)], - { - requestedLimit: 4, - }, + {}, ); expect(paginator.getItem('m4')).toBeUndefined(); // within-window delete removed expect(paginator.getItem('m1')).toBeDefined(); // older-than-page kept expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm5', 'm6']); }); - it('removes the OLDEST loaded message once the page proves it reached the channel start', () => { - // Whole channel loaded. m1 (oldest) hard-deleted → a request for four returns only three. + it('keeps a below-window delete even when the page proves reached-start — reconcile is window-only (data-loss safe)', () => { + // m1 (oldest) hard-deleted → a bounded re-fetch (requestedLimit 4 <= pageSize) returns [m2,m3,m4], + // short, which DOES prove reached-start (hasMoreTail goes false). But m1 sits BELOW the returned + // window, and reconcile is window-only — it never removes anything older than the page's oldest — + // so m1 is KEPT rather than risk deleting a merely-not-fetched message; the stale ghost self-heals + // on a cold load. requestedLimit moved only the flag, never a deletion. const paginator = loadHead( [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], - { - isTail: true, - }, + { isTail: true }, ); paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], { requestedLimit: 4, }); - expect(paginator.getItem('m1')).toBeUndefined(); - expect(ids(paginator)).toEqual(['m2', 'm3', 'm4']); + expect(paginator.getItem('m1')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4']); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); }); it('keeps the oldest loaded message when the page did NOT prove it reached the start', () => { @@ -2050,9 +2050,7 @@ describe('MessagePaginator', () => { }, ); // A FULL page (returned === requested) that simply does not reach m1 → cannot claim m1 deleted. - paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], { - requestedLimit: 3, - }); + paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], {}); expect(paginator.getItem('m1')).toBeDefined(); expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4']); }); @@ -2070,9 +2068,7 @@ describe('MessagePaginator', () => { // m5 (newest) deleted; the page's newest is now m4. No snapshot ⇒ the top edge is ambiguous. paginator.mergeNewestPage( [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], - { - requestedLimit: 5, - }, + {}, ); expect(paginator.getItem('m5')).toBeDefined(); }); @@ -2093,7 +2089,6 @@ describe('MessagePaginator', () => { [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], { candidateIds, - requestedLimit: 5, }, ); @@ -2115,7 +2110,6 @@ describe('MessagePaginator', () => { paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { candidateIds, - requestedLimit: 5, }); expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); @@ -2145,7 +2139,6 @@ describe('MessagePaginator', () => { [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], { candidateIds, - requestedLimit: 5, }, ); @@ -2171,7 +2164,6 @@ describe('MessagePaginator', () => { // The server page only has the confirmed m1 + m5; the local-only ones the server never saw. paginator.mergeNewestPage([msg('m1', 1), msg('m5', 5)], { candidateIds, - requestedLimit: 5, }); expect(paginator.getItem('sending')).toBeDefined(); @@ -2194,7 +2186,6 @@ describe('MessagePaginator', () => { // m3 hard-deleted; the failed send was never on the server. Page: [m1, m4]. paginator.mergeNewestPage([msg('m1', 1), msg('m4', 4)], { candidateIds, - requestedLimit: 4, }); expect(paginator.getItem('m3')).toBeUndefined(); @@ -2209,7 +2200,7 @@ describe('MessagePaginator', () => { const paginator = loadHead(loaded); const candidateIds = new Set(loaded.map((message) => message.id)); - paginator.mergeNewestPage([], { candidateIds, requestedLimit: 3 }); + paginator.mergeNewestPage([], { candidateIds }); expect(paginator.items).toEqual([]); expect(paginator.lastMessage).toBeNull(); @@ -2228,7 +2219,7 @@ describe('MessagePaginator', () => { // A live message arrives during the fetch, then the (stale) empty page comes back. paginator.ingestItem(msg('m3', 3)); - paginator.mergeNewestPage([], { candidateIds, requestedLimit: 2 }); + paginator.mergeNewestPage([], { candidateIds }); expect(paginator.getItem('m1')).toBeUndefined(); expect(paginator.getItem('m2')).toBeUndefined(); @@ -2246,7 +2237,6 @@ describe('MessagePaginator', () => { // A fully-disjoint newest window (100+ arrived). Rebuild replaces the loaded set; no extra prune. paginator.mergeNewestPage([msg('m10', 10), msg('m11', 11), msg('m12', 12)], { candidateIds, - requestedLimit: 3, }); expect(ids(paginator)).toEqual(['m10', 'm11', 'm12']); @@ -2275,7 +2265,6 @@ describe('MessagePaginator', () => { // Active window is the older [m1,m2,m3]; the head holds m8,m9,m10 (m9 "deleted" server-side). paginator.mergeNewestPage([msg('m8', 8), msg('m10', 10)], { candidateIds: new Set(['m8', 'm9', 'm10']), - requestedLimit: 3, }); // The view (the older window) is preserved — no yank to the head — but the hidden-head ghost m9 @@ -2290,49 +2279,53 @@ describe('MessagePaginator', () => { paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3), msg('m4', 4)], { candidateIds: new Set(loaded.map((message) => message.id)), - requestedLimit: 4, }); expect(ids(paginator)).toEqual(['m1', 'm3', 'm4']); paginator.mergeNewestPage([msg('m1', 1), msg('m3', 3), msg('m4', 4)], { candidateIds: new Set(['m1', 'm3', 'm4']), - requestedLimit: 4, }); expect(ids(paginator)).toEqual(['m1', 'm3', 'm4']); }); - // ── The "100 point" is data-driven, never hardcoded: reconcile only within the returned page ── + // ── Reconcile is window-only: nothing older than the returned page's oldest is ever removed (no cap) ── - it('never reconciles messages older than what the page returns, even on an over-request (clamp)', () => { - // 105 loaded (whole channel). The server caps its response at 100 (the newest 100). We requested - // more (105) but only 100 come back — that shortfall must NOT be read as "reached the channel - // start", or the 5 oldest (beyond the page) would be wrongly deleted. + it('keeps every loaded message older than the returned page — window-only reconcile never deletes below it (data-loss guard)', () => { + // DATA-LOSS GUARD. 105 loaded; a reload over-requests all 105 but the server caps the page at the + // newest 100 — the 5 oldest sit BELOW the returned page's oldest. Reconcile is window-only: it + // never removes anything older than the returned page's oldest, so none of the 5 are deleted. + // requestedLimit only tunes hasMoreTail, never a deletion — the window-only reconcile holds for ANY + // page that covers only part of the loaded window. const all = Array.from({ length: 105 }, (_, i) => msg(`msg-${String(i).padStart(3, '0')}`, i), ); const paginator = loadHead(all, { isTail: true }); - const page = all.slice(5); // the newest 100 — the server's capped response + const page = all.slice(5); // a page covering only the newest 100 of the 105 loaded paginator.mergeNewestPage(page, { + requestedLimit: all.length, // reload asks for all 105; the server caps the returned page at 100 candidateIds: new Set(all.map((message) => message.id)), - requestedLimit: 105, }); - // All 105 kept: nothing was actually deleted, and the 5 oldest are beyond the page's reach. + // All 105 kept: nothing was actually deleted, and the 5 oldest are below the returned page. expect(paginator.items?.length).toBe(105); expect(paginator.getItem('msg-000')).toBeDefined(); expect(paginator.getItem('msg-004')).toBeDefined(); }); - it('over-request: a server-capped page keeps hasMoreTail and anchors the tail cursor to the true loaded oldest', () => { + it('an over-request capped to the newest part keeps hasMoreTail and anchors the tail cursor to the true loaded oldest', () => { + // A reload over-requests all 105 but the server caps the page at the newest 100. An over-request + // (105 > pageSize 100) can be silently server-capped, so a short page CANNOT prove reached-start — + // hasMoreTail stays true and the tail cursor anchors to the true oldest LOADED (msg-000), not the + // page's oldest — so "load older" resumes contiguously from the real bottom of the window. const all = Array.from({ length: 105 }, (_, i) => msg(`msg-${String(i).padStart(3, '0')}`, i), ); const paginator = loadHead(all, { isTail: true }); paginator.mergeNewestPage(all.slice(5), { + requestedLimit: all.length, candidateIds: new Set(all.map((message) => message.id)), - requestedLimit: 105, }); const state = paginator.state.getLatestValue(); @@ -2340,11 +2333,12 @@ describe('MessagePaginator', () => { expect(state.cursor?.tailward).toBe('msg-000'); }); - it('reached channel start: a page shorter than the clamped limit clears hasMoreTail, nulls the tail cursor and sets isTail', () => { - // The complementary branch: the loaded window believes older messages exist (isTail:false → - // hasMoreTail true), then the newest two are hard-deleted so the reconnect page comes back short - // of the requested limit. A short page (3 < min(5,100)) proves we reached the channel start, so - // hasMoreTail drops to false, the tail cursor nulls, and the interval's isTail flips true. + it('a bounded short page reaches the channel start: trailing deletes still removed, hasMoreTail false, cursor cleared', () => { + // The newest two are hard-deleted so the reconnect page comes back short. The trailing deletes + // (m4, m5) are still removed — they are at/above the newest returned message and the pre-fetch + // snapshot proves them gone. The request was BOUNDED (requestedLimit 5 <= pageSize 100) and came + // back short, so it DOES prove reached-start: hasMoreTail goes false, the interval's isTail goes + // true, and the tail cursor is cleared. No spurious "load older". const paginator = loadHead([ msg('m1', 1), msg('m2', 2), @@ -2356,17 +2350,268 @@ describe('MessagePaginator', () => { // m4 + m5 hard-deleted while offline: only the surviving newest come back. paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { - candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), requestedLimit: 5, + candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), }); const state = paginator.state.getLatestValue(); - expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); // trailing deletes removed via snapshot expect(state.hasMoreTail).toBe(false); - expect(state.cursor?.tailward).toBeNull(); + expect(state.cursor?.tailward).toBe(null); expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(true); }); + // ── hasMoreTail derivation: bounded (reliable) vs over-request (conservative), no hardcoded cap ── + + describe('hasMoreTail derivation (requestedLimit vs pageSize)', () => { + it('a bounded short page proves reached-start → hasMoreTail false, cursor cleared (no spurious "load older")', () => { + // The channel is smaller than a page: a bounded open (requestedLimit <= pageSize) returns every + // message and comes back short. That reliably proves reached-start — pageSize <= the server's max + // page size is the same invariant executeQuery pagination relies on — so hasMoreTail is false. + // This is the fix for the spurious top spinner + double pagination on first open. + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { + requestedLimit: 25, + }); + const state = paginator.state.getLatestValue(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3']); // nothing removed + expect(state.hasMoreTail).toBe(false); + expect(state.cursor?.tailward).toBe(null); + expect((paginator.itemIntervals[0] as { isTail?: boolean }).isTail).toBe(true); + }); + + it('a bounded FULL page does NOT assert reached-start → hasMoreTail true (older may remain)', () => { + // A bounded request that comes back FULL (length === requested) means older messages may still + // exist, so "load older" stays enabled and the cursor anchors to the loaded oldest. + const paginator = loadHead([msg('m1', 1), msg('m2', 2), msg('m3', 3)]); + paginator.mergeNewestPage([msg('m1', 1), msg('m2', 2), msg('m3', 3)], { + requestedLimit: 3, + }); + const state = paginator.state.getLatestValue(); + expect(state.hasMoreTail).toBe(true); + expect(state.cursor?.tailward).toBe('m1'); + }); + + it('an OVER-request short page (> pageSize, possibly server-capped) never asserts reached-start → hasMoreTail true, below-window kept', () => { + // reload re-fetches the whole loaded window to reconcile as much as possible; that over-request + // (> pageSize) can be silently server-capped, so a short page proves nothing about reaching the + // start. Bias to true so a merely-capped page is never mistaken for the channel's start — the + // data-loss-safe direction, needing no hardcoded max page size. + const paginator = new MessagePaginator({ + channel, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + paginatorOptions: { pageSize: 3 }, + }); + paginator.ingestPage({ + page: [msg('m1', 1), msg('m2', 2), msg('m3', 3), msg('m4', 4)], + isHead: true, + isTail: true, + setActive: true, + }); + // reload over-requests 4 (> pageSize 3); the server caps the returned page at the newest 3. + paginator.mergeNewestPage([msg('m2', 2), msg('m3', 3), msg('m4', 4)], { + requestedLimit: 4, + }); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); + expect(paginator.getItem('m1')).toBeDefined(); // below-window delete never removed (data-loss safe) + }); + }); + + // ── reached-start probe: the ONLY cap-free way to prune the oldest-run below the returned window ── + + describe('reached-start probe (below-window reconciliation)', () => { + const mockQuery = () => channel.query as unknown as ReturnType; + const mockGetReplies = () => + channel.getReplies as unknown as ReturnType; + const flushProbe = (p: MessagePaginator) => + (p as unknown as { _belowWindowReconcile?: Promise })._belowWindowReconcile; + const makePaginator = (pageSize: number, parentMessageId?: string) => + new MessagePaginator({ + channel, + parentMessageId, + itemIndex: new StoreBackedItemIndex({ + getEntityId: (message) => message.id, + }), + paginatorOptions: { pageSize }, + }); + // Fully-loaded head (isTail → hasMoreTail false, so condition A holds). + const loadFull = (paginator: MessagePaginator, page: LocalMessage[]) => + paginator.ingestPage({ page, isHead: true, isTail: true, setActive: true }); + + it('probe empty → prunes the oldest-run ghost below the window and settles hasMoreTail', async () => { + // pageSize 3 so requestedLimit 5 is an OVER-request: the sync merge cannot prove reached-start + // (biases hasMoreTail true, keeps m1 window-only) — the probe is the SOLE driver here. + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); // A precondition + + // m1 (oldest) hard-deleted offline; reload over-requests all 5, server returns the survivors. + mockQuery().mockResolvedValue({ messages: [] }); // nothing older than m2 → reached start + paginator.mergeNewestPage( + [msg('m2', 2), msg('m3', 3), msg('m4', 4), msg('m5', 5)], + { requestedLimit: 5, candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']) }, + ); + // Sync merge kept m1 (window-only) and biased hasMoreTail true (over-request): + expect(paginator.getItem('m1')).toBeDefined(); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); + + await flushProbe(paginator); + + expect(channel.query).toHaveBeenCalledWith({ + messages: { limit: 1, id_lt: 'm2' }, + }); + expect(paginator.getItem('m1')).toBeUndefined(); // pruned by the probe + expect(ids(paginator)).toEqual(['m2', 'm3', 'm4', 'm5']); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); // settled + expect(paginator.state.getLatestValue().cursor?.tailward).toBe(null); + }); + + it('probe returns an older message → keeps the below-window items (truncated, not the start)', async () => { + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + // Reload over-requests 5; server caps and returns only the newest 3 — m1,m2 fall below. + mockQuery().mockResolvedValue({ messages: [msg('m2', 2)] }); // older content exists + paginator.mergeNewestPage([msg('m3', 3), msg('m4', 4), msg('m5', 5)], { + requestedLimit: 5, + candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), + }); + + await flushProbe(paginator); + + expect(channel.query).toHaveBeenCalledWith({ + messages: { limit: 1, id_lt: 'm3' }, + }); + expect(paginator.getItem('m1')).toBeDefined(); // kept — no data loss on a truncated reload + expect(paginator.getItem('m2')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4', 'm5']); + }); + + it('does NOT probe when we were not at the channel start (hasMoreTail was true)', async () => { + const paginator = makePaginator(3); + // isTail false → hasMoreTail true → condition A fails. + paginator.ingestPage({ + page: [msg('m3', 3), msg('m4', 4), msg('m5', 5)], + isHead: true, + isTail: false, + setActive: true, + }); + expect(paginator.state.getLatestValue().hasMoreTail).toBe(true); + paginator.mergeNewestPage([msg('m4', 4), msg('m5', 5)], { + requestedLimit: 3, + candidateIds: new Set(['m3', 'm4', 'm5']), + }); + await flushProbe(paginator); + expect(channel.query).not.toHaveBeenCalled(); + expect(paginator.getItem('m3')).toBeDefined(); + }); + + it('does NOT probe a small-page caller (requestedLimit < loaded) — a list hydrate cannot reach the start', async () => { + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + // A list-hydrate-style page: asked for only 2, far fewer than the 5 loaded. + paginator.mergeNewestPage([msg('m4', 4), msg('m5', 5)], { + requestedLimit: 2, + candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), + }); + await flushProbe(paginator); + expect(channel.query).not.toHaveBeenCalled(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm4', 'm5']); // nothing pruned + }); + + it('does NOT probe when the oldest is still in the page (a middle delete — within-span handles it)', async () => { + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + // m3 deleted; reload returns [m1,m2,m4,m5] — the oldest (m1) is still present. + paginator.mergeNewestPage( + [msg('m1', 1), msg('m2', 2), msg('m4', 4), msg('m5', 5)], + { + requestedLimit: 5, + candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']), + }, + ); + await flushProbe(paginator); + expect(channel.query).not.toHaveBeenCalled(); + expect(paginator.getItem('m3')).toBeUndefined(); // removed by within-span, not the probe + expect(ids(paginator)).toEqual(['m1', 'm2', 'm4', 'm5']); + }); + + it('threads probe via getReplies', async () => { + const paginator = makePaginator(3, 'parent-1'); + const reply = (id: string, minute: number) => + msg(id, minute, { parent_id: 'parent-1' }); + loadFull(paginator, [ + reply('r1', 1), + reply('r2', 2), + reply('r3', 3), + reply('r4', 4), + ]); + mockGetReplies().mockResolvedValue({ messages: [] }); // nothing older than r2 + paginator.mergeNewestPage([reply('r2', 2), reply('r3', 3), reply('r4', 4)], { + requestedLimit: 4, + candidateIds: new Set(['r1', 'r2', 'r3', 'r4']), + }); + await flushProbe(paginator); + expect(channel.getReplies).toHaveBeenCalledWith({ + parent_id: 'parent-1', + limit: 1, + id_lt: 'r2', + }); + expect(channel.query).not.toHaveBeenCalled(); + expect(paginator.getItem('r1')).toBeUndefined(); + }); + + it('aborts the prune if the head interval changed while the probe was in flight', async () => { + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + let resolveProbe: () => void = () => undefined; + mockQuery().mockReturnValue( + new Promise((resolve) => { + resolveProbe = () => resolve({ messages: [] }); + }), + ); + paginator.mergeNewestPage( + [msg('m2', 2), msg('m3', 3), msg('m4', 4), msg('m5', 5)], + { requestedLimit: 5, candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']) }, + ); + // The head interval goes away (a reset/jump) before the probe resolves. + paginator.setIntervals([]); + resolveProbe(); + await expect(flushProbe(paginator)).resolves.toBeUndefined(); // no throw, guard bailed + }); + }); + describe('batch({ coalesce: true }) — single deterministic window publish', () => { it('coalesces N removals into a single state publish', () => { const paginator = loadHead([ @@ -2463,7 +2708,6 @@ describe('MessagePaginator', () => { paginator.mergeNewestPage(page, { candidateIds: new Set(all.map((message) => message.id)), - requestedLimit: 105, }); expect(paginator.getItem('msg-050')).toBeUndefined(); // within-page delete removed @@ -2566,8 +2810,10 @@ describe('MessagePaginator', () => { }); expect(paginator.state.getLatestValue().hasMoreTail).toBe(false); - // Reconnect fetches a FULL page (requestedLimit == page length) → older messages remain, so - // hasMoreTail must be RE-COMPUTED from the page (not read off the stale interval flag). + // Reconnect re-seeds the head window. The merge biases hasMoreTail to `true` (isTail:false), + // which clears the stale "complete" flag so "load older" works again — not read off the stale + // interval flag. (If the channel really is fully loaded, the next load-older returns empty and + // settles it.) paginator.seedFirstPageSync( [msg('m1', 1), msg('m2', 2), msg('m3', 3)], 3, @@ -2816,7 +3062,6 @@ describe('MessagePaginator', () => { const candidateIds = new Set(paginator.headItems.map((m) => m.id)); paginator.mergeNewestPage([msg('m90', 90), msg('m95', 95)], { candidateIds, - requestedLimit: 3, }); // View preserved (still on the island)... From 20bb4d3920f31e0a25e6ca59d2ea7fc792dca68d Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Thu, 13 Aug 2026 16:14:29 +0200 Subject: [PATCH 16/16] chore: add test for both removal and receival --- .../paginators/MessagePaginator.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/unit/pagination/paginators/MessagePaginator.test.ts b/test/unit/pagination/paginators/MessagePaginator.test.ts index 5c2b94798..c2f08fbc0 100644 --- a/test/unit/pagination/paginators/MessagePaginator.test.ts +++ b/test/unit/pagination/paginators/MessagePaginator.test.ts @@ -2610,6 +2610,38 @@ describe('MessagePaginator', () => { resolveProbe(); await expect(flushProbe(paginator)).resolves.toBeUndefined(); // no throw, guard bailed }); + + it('trailing deletes covered by new arrivals become within-span: removes them, merges the new, keeps the below-window oldest', async () => { + // Loaded fully before going offline. + const paginator = makePaginator(3); + loadFull(paginator, [ + msg('m1', 1), + msg('m2', 2), + msg('m3', 3), + msg('m4', 4), + msg('m5', 5), + ]); + + mockQuery().mockResolvedValue({ messages: [msg('m1', 1)] }); // probe: m1 IS older than m2 → keep it + paginator.mergeNewestPage( + [msg('m2', 2), msg('m3', 3), msg('m6', 6), msg('m7', 7), msg('m8', 8)], + { requestedLimit: 5, candidateIds: new Set(['m1', 'm2', 'm3', 'm4', 'm5']) }, + ); + + // Sync: m4/m5 within-span → gone; m6/m7/m8 merged; m1 below the window kept pending the probe. + expect(paginator.getItem('m4')).toBeUndefined(); + expect(paginator.getItem('m5')).toBeUndefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm6', 'm7', 'm8']); + + await flushProbe(paginator); + + // The below-window oldest (m1) was probed and is real → kept. Final = the server truth. + expect(channel.query).toHaveBeenCalledWith({ + messages: { limit: 1, id_lt: 'm2' }, + }); + expect(paginator.getItem('m1')).toBeDefined(); + expect(ids(paginator)).toEqual(['m1', 'm2', 'm3', 'm6', 'm7', 'm8']); + }); }); describe('batch({ coalesce: true }) — single deterministic window publish', () => {