From ae8c7a53632f5776904cb2bad994f18df3cc9a6a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 30 Jul 2026 22:34:58 -0700 Subject: [PATCH 01/15] feat(files): stream copilot edits into the collaborative doc smoothly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - apply the agent stream client-side into the live Yjs binding as minimal updateYFragment diffs (like main's setContent, but incremental) so it renders smoothly AND broadcasts to every peer via CRDT — a collaborator on /files sees the stream for free - gate the apply on collabReady so diffs never land on an unseeded doc; keep the read-only placeholder visible until the seed swaps in - run streamed ops under a dedicated tx origin so they stay out of the user's undo stack - delete the throttled server-side streaming merge and the baseVersion ordering machinery it needed (relay + notify + session contract); the durable final write still reconciles open editors and seeds late joiners --- .../handlers/file-doc.multireplica.test.ts | 20 +- apps/realtime/src/handlers/file-doc.test.ts | 33 ---- apps/realtime/src/handlers/file-doc.ts | 37 +--- apps/realtime/src/routes/http.ts | 12 +- .../apply-streamed-markdown.test.ts | 125 ++++++++++++ .../collaboration/apply-streamed-markdown.ts | 32 ++++ .../rich-markdown-editor.tsx | 114 ++++++++--- .../request/go/file-preview-adapter.test.ts | 178 +++--------------- .../request/go/file-preview-adapter.ts | 73 +------ .../session/file-preview-session-contract.ts | 4 - .../request/session/file-preview-session.ts | 2 - .../tools/server/files/file-preview.ts | 10 +- apps/sim/lib/realtime/notify.test.ts | 74 +++----- apps/sim/lib/realtime/notify.ts | 68 +++---- 14 files changed, 353 insertions(+), 429 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts diff --git a/apps/realtime/src/handlers/file-doc.multireplica.test.ts b/apps/realtime/src/handlers/file-doc.multireplica.test.ts index 17c17ffe371..df56ffdc4c0 100644 --- a/apps/realtime/src/handlers/file-doc.multireplica.test.ts +++ b/apps/realtime/src/handlers/file-doc.multireplica.test.ts @@ -60,7 +60,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(new Y.Doc())) }) - it('drops a stale-base streaming snapshot against the SHARED synced version and never records it', async () => { + it('drops a stale durable write against the SHARED synced version', async () => { // A durable write (e.g. a concurrent human save on another process) records the shared synced version. expect(await applyMarkdownToLiveFileDoc('file-1', '# durable', { version: 100 })).toBe( 'applied' @@ -68,23 +68,19 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) mockFetchFileDocMerge.mockClear() - // A streaming snapshot built from an older base (50) than the SHARED synced version is stale — - // rejected under the lock before any diff is built, so it can't clobber the durable write. - expect( - await applyMarkdownToLiveFileDoc('file-1', '# stale-base stream', { baseVersion: 50 }) - ).toBe('stale') + // A durable write with an OLDER version than the SHARED synced version is stale — rejected under the + // lock before any diff is built, so it can't regress the doc across replicas. + expect(await applyMarkdownToLiveFileDoc('file-1', '# older durable', { version: 50 })).toBe( + 'stale' + ) expect(mockFetchFileDocMerge).not.toHaveBeenCalled() - // A streaming snapshot whose base is the current shared version applies (nothing newer to clobber)... - expect( - await applyMarkdownToLiveFileDoc('file-1', '# current-base stream', { baseVersion: 100 }) - ).toBe('applied') - // ...but is never recorded: a later durable write at 150 still applies. + // A newer durable write applies and advances the shared synced version. expect(await applyMarkdownToLiveFileDoc('file-1', '# durable again', { version: 150 })).toBe( 'applied' ) expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150) - // setSyncedVersion fired only for the two durable writes, never for a streaming snapshot. + // setSyncedVersion fired only for the two applied durable writes, never for the stale one. expect(fakeStore.setSyncedVersion).toHaveBeenCalledTimes(2) }) }) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 7ccbe3d2254..a6f59df67bf 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -569,39 +569,6 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocMerge).not.toHaveBeenCalled() }) - it('drops a streaming snapshot whose base predates a newer durable write, but never records it', async () => { - mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original')) // seed version 1 - const { io } = createIo() - const { handlers } = setup('socket-1', io) - await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) - await flushMicrotasks() - - mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(new Y.Doc())) - - // A durable write (e.g. a concurrent human save) lands and is recorded as the synced version. - expect(await applyMarkdownToLiveFileDoc('file-1', '# durable', { version: 100 })).toBe( - 'applied' - ) - - // A streaming snapshot built from an OLDER base (50) — copilot loaded the file before that durable - // write — is stale: applying it would diff the live doc back toward the copilot content and clobber - // the durable write, which a later persist would then write over the file. - expect( - await applyMarkdownToLiveFileDoc('file-1', '# stale-base stream', { baseVersion: 50 }) - ).toBe('stale') - - // A streaming snapshot whose base IS the current durable version applies — nothing newer to clobber. - expect( - await applyMarkdownToLiveFileDoc('file-1', '# current-base stream', { baseVersion: 100 }) - ).toBe('applied') - - // ...and a streaming merge is never recorded as the synced version: a later durable write at 150 still - // applies (only durable writes move the synced version; the final edit_content write reconciles). - expect(await applyMarkdownToLiveFileDoc('file-1', '# durable again', { version: 150 })).toBe( - 'applied' - ) - }) - it('serializes concurrent merges for the same file (second waits for the first)', async () => { mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original')) const { io } = createIo() diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 039e2ab6661..7e502fc0582 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -517,12 +517,11 @@ const fileDocMergeChains = new Map>() /** * How a merge is positioned on the file's version line — mirrors the sim-side `LiveFileDocMergeOrder` - * wire fields. A durable `version` is checked AND recorded; a streaming `baseVersion` (the durable version - * the snapshot was built from) is checked only — dropped if a newer durable write has since landed. + * wire field. A durable `version` is checked (applied only if newer than the doc's current version) AND + * recorded as the synced version. */ interface MergeOrder { version?: number - baseVersion?: number } /** @@ -569,16 +568,14 @@ async function mergeMarkdownIntoRoom( name: string, fileId: string, markdown: string, - { version, baseVersion }: MergeOrder + { version }: MergeOrder ): Promise<'applied' | 'no-live-room' | 'merge-unavailable' | 'stale'> { const store = getFileDocStore() // The durable version this merge carries is now incorporated in the live doc — record it (cluster-wide // in Redis for multi-task, plus this task's room) so the persist If-Match guard treats this write as // synced rather than an out-of-band conflict. AWAITED so the version is durable before the merge lock - // releases, so the next lock holder's staleness check (below) reads a consistent value. Only a durable - // `version` is recorded — a streaming `baseVersion` orders the merge (below) but is never a checkpoint, - // so the synced version stays pinned to the last durable write. + // releases, so the next lock holder's staleness check (below) reads a consistent value. const recordVersion = async () => { if (version === undefined) return const room = fileDocRooms.get(name) @@ -590,29 +587,13 @@ async function mergeMarkdownIntoRoom( } // Order this merge on the file's version line, where `current` is the durable version the doc already - // incorporates. Both keys are DB-monotonic `contentUpdatedAt` values (no wall-clock), so ordering is - // immune to clock skew: - // - A durable `version` is stale if it is NOT strictly newer than `current` — a newer durable write - // already landed (possibly on another process, out of dispatch order); applying its older markdown - // would regress the doc while the monotonic token stays high. - // - A streaming `baseVersion` (the durable version the snapshot was built from) is stale if `current` - // has moved PAST it — a newer durable write landed since the snapshot's base, so diffing the live - // doc back toward the snapshot would clobber that write's content (which a later persist, still - // holding the current If-Match token, would then write over the durable file). This is what stops a - // concurrent human edit from being silently lost; the per-process caller chain cannot see it. - // A merge with neither key is never stale (legacy, unordered). Only a durable `version` is recorded, - // so a streaming snapshot never advances the synced version — the final `edit_content` write does. - // - // Known, accepted limitation: two INDEPENDENT copilot streams editing the SAME file at once share one - // base version, so neither is stale relative to the other and their snapshots can interleave in the live - // doc. This is transient only — each stream's final durable write is version-ordered and reconciles the - // doc, so the steady state is deterministic (last durable wins) and the durable file is never corrupted. - // Ordering two independent snapshot streams would need a shared sequence they don't have; the fully - // robust form (a per-file streaming lease, or embedding the version in each stream entry) is a scoped - // follow-up, not a durability fix owed here. + // incorporates. `version` is a DB-monotonic `contentUpdatedAt` value (no wall-clock), so ordering is + // immune to clock skew: a durable `version` is stale if it is NOT strictly newer than `current` — a + // newer durable write already landed (possibly on another process, out of dispatch order); applying its + // older markdown would regress the doc while the monotonic token stays high. A merge with no `version` + // is never stale (legacy, unordered). const isStale = (current: number): boolean => { if (version !== undefined) return version <= current - if (baseVersion !== undefined) return current > baseVersion return false } diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index b37e9061107..e81ed3150ad 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -203,22 +203,22 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } - // Merge a copilot edit into a file's LIVE collaborative document so it streams into open editors - // (Stage C). Returns `{ applied }`: when false, no seeded live room exists and the caller writes - // the file directly instead. Any live user edits are preserved — the app builds a minimal CRDT diff. + // Merge a durable file write into a file's LIVE collaborative document so open editors reconcile to + // it (Stage C) — this is the stream-end/durable reconcile, not token-by-token streaming (that is now + // applied client-side by the open editor). Returns `{ applied }`: when false, no seeded live room + // exists and the caller writes the file directly instead. Live user edits are preserved — the app + // builds a minimal CRDT diff. if (req.method === 'POST' && req.url === '/api/file-doc/apply-edit') { try { const body = await readRequestBody(req) - const { fileId, markdown, version, baseVersion } = JSON.parse(body) + const { fileId, markdown, version } = JSON.parse(body) if (!isNonEmptyString(fileId) || typeof markdown !== 'string') { return sendError(res, 'Invalid fileId or markdown', 400) } // `version` (the durable updatedAt this markdown was written with) records that the live doc now // incorporates that durable version, so the persist If-Match guard won't flag it as a conflict. - // `baseVersion` is a streaming snapshot's causal base: dropped if a newer durable write landed. const result = await applyMarkdownToLiveFileDoc(fileId, markdown, { version: typeof version === 'number' ? version : undefined, - baseVersion: typeof baseVersion === 'number' ? baseVersion : undefined, }) res.writeHead(200, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ applied: result === 'applied' })) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts new file mode 100644 index 00000000000..c9aa02f732f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts @@ -0,0 +1,125 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { afterEach, beforeAll, describe, expect, it } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { createMarkdownEditorExtensions } from '../editor-extensions' +import { applyStreamedMarkdownToLiveDoc } from './apply-streamed-markdown' + +beforeAll(() => { + // jsdom does not implement elementFromPoint; the Placeholder extension's viewport tracking calls it + // on view mount. Returning null makes ProseMirror's posAtCoords fall back gracefully. + if (!document.elementFromPoint) { + document.elementFromPoint = () => null + } +}) + +/** A headless collaborative editor bound to a fresh Y.Doc — the same extension wiring the component uses. */ +function makeCollabEditor() { + const doc = new Y.Doc() + const awareness = new Awareness(doc) + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { + doc, + awareness, + user: { name: 'Tester', color: '#ffffff', clientId: doc.clientID }, + }, + }), + content: '', + }) + return { editor, doc, awareness } +} + +const teardown: Array<() => void> = [] +afterEach(() => { + for (const fn of teardown.splice(0)) fn() +}) + +function track(t: { editor: Editor; doc: Y.Doc; awareness: Awareness }) { + teardown.push(() => { + t.editor.destroy() + t.awareness.destroy() + t.doc.destroy() + }) + return t +} + +describe('applyStreamedMarkdownToLiveDoc', () => { + it('streams agent content into the live collaborative doc and broadcasts it as Yjs ops', () => { + const { editor, doc } = track(makeCollabEditor()) + + expect(applyStreamedMarkdownToLiveDoc(editor, '# Title\n\nHello world.')).toBe(true) + expect(editor.getText()).toContain('Hello world') + + // The write lands as ops on the shared doc, so any peer receives it (this is what makes a + // collaborator on /files see the stream without ever holding `streamingContent`). + const peer = new Y.Doc() + Y.applyUpdate(peer, Y.encodeStateAsUpdate(doc)) + expect(peer.getXmlFragment('default').toString()).toContain('Hello world') + peer.destroy() + }) + + it('returns false when the doc is not yet bound (no ySync binding)', () => { + // A plain editor with no collaboration has no ySync binding, so the applier reports "not ready" + // rather than throwing — the streaming tick re-arms until the doc is seeded. + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ placeholder: '' }), + content: '', + }) + teardown.push(() => editor.destroy()) + expect(applyStreamedMarkdownToLiveDoc(editor, '# Nope')).toBe(false) + }) + + it('keeps agent-streamed ops out of the undo stack while user edits stay undoable', () => { + const { editor } = track(makeCollabEditor()) + + applyStreamedMarkdownToLiveDoc(editor, '# Streamed\n\nAgent wrote this.') + // The streamed op used AGENT_STREAM_ORIGIN, which the Collaboration UndoManager does not track — + // so there is nothing to undo, and an undo must not revert the agent's content. + expect(editor.can().undo()).toBe(false) + editor.commands.undo() + expect(editor.getText()).toContain('Agent wrote this') + + // A genuine user edit IS captured (origin ySyncPluginKey) — proving the test isn't vacuous: + // undo works, and it reverts only the user edit, leaving the agent content intact. + editor.commands.focus('end') + editor.commands.insertContent(' USER-TYPED') + expect(editor.getText()).toContain('USER-TYPED') + expect(editor.can().undo()).toBe(true) + editor.commands.undo() + expect(editor.getText()).not.toContain('USER-TYPED') + expect(editor.getText()).toContain('Agent wrote this') + }) + + it('merges an agent write with a concurrent peer edit (minimal diff, no clobber)', () => { + const { editor, doc } = track(makeCollabEditor()) + applyStreamedMarkdownToLiveDoc(editor, 'Alpha paragraph.\n\nBeta paragraph.') + + // A peer forks the current state and edits the FIRST paragraph directly on the shared type… + const remote = new Y.Doc() + Y.applyUpdate(remote, Y.encodeStateAsUpdate(doc)) + const remoteFrag = remote.getXmlFragment('default') + remote.transact(() => { + const firstPara = remoteFrag.get(0) as Y.XmlElement + const textNode = firstPara.get(0) as Y.XmlText + textNode.insert(textNode.toString().length, ' EDITED') + }) + + // …while the agent rewrites the SECOND paragraph through the live editor binding. + applyStreamedMarkdownToLiveDoc(editor, 'Alpha paragraph.\n\nBeta paragraph, expanded.') + + // Exchange updates both ways (as the relay would); a full-document replace would have clobbered + // the peer's concurrent edit — a minimal `updateYFragment` diff preserves both. + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote, Y.encodeStateVector(doc))) + Y.applyUpdate(remote, Y.encodeStateAsUpdate(doc, Y.encodeStateVector(remote))) + + const merged = doc.getXmlFragment('default').toString() + expect(merged).toContain('EDITED') + expect(merged).toContain('expanded') + remote.destroy() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts new file mode 100644 index 00000000000..a92f11110d6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts @@ -0,0 +1,32 @@ +import type { Editor } from '@tiptap/core' +import { Node as PMNode } from '@tiptap/pm/model' +import { updateYFragment, ySyncPluginKey } from '@tiptap/y-tiptap' +import { parseMarkdownToDoc } from '../markdown-parse' + +/** + * Transaction origin for agent-streamed writes into a live collaborative doc. It is deliberately NOT + * the `ySyncPluginKey` origin that local user edits use, so the Collaboration UndoManager — which + * tracks only `ySyncPluginKey` — excludes streamed ops from the user's undo stack. + */ +const AGENT_STREAM_ORIGIN = Symbol('agent-stream') + +/** + * Apply a streamed markdown body into the editor's live collaborative Y.Doc as a minimal CRDT diff. + * + * Uses the running `ySyncPlugin` binding's {@link updateYFragment} — the same primitive TipTap runs on + * every keystroke — so only the delta between the doc's current content and `body` is written, never a + * full-document replace that would wipe collaborators. Each diff is a small Yjs op that renders locally + * (via the binding's observer, the remote-edit render path) AND broadcasts to every peer, so the stream + * is smooth here and on other clients alike. Runs under {@link AGENT_STREAM_ORIGIN} so the streamed ops + * stay out of the user's undo stack. Returns `false` when the editor has no live ySync binding (e.g. a + * non-collaborative editor); the caller gates seed-readiness separately via `collabReady`. + */ +export function applyStreamedMarkdownToLiveDoc(editor: Editor, body: string): boolean { + const binding = ySyncPluginKey.getState(editor.state)?.binding + if (!binding) return false + const target = PMNode.fromJSON(editor.schema, parseMarkdownToDoc(body)) + binding.doc.transact(() => { + updateYFragment(binding.doc, binding.type, target, binding) + }, AGENT_STREAM_ORIGIN) + return true +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index f2a1eaad79b..540fc955301 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -20,6 +20,7 @@ import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' import { PreviewLoadingFrame } from '../preview-shared' import { useEditableFileContent } from '../use-editable-file-content' +import { applyStreamedMarkdownToLiveDoc } from './collaboration/apply-streamed-markdown' import { useFileDocCollaboration } from './collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from './editor-extensions' import { findHeadingPos } from './heading-anchors' @@ -83,10 +84,10 @@ interface RichMarkdownEditorProps { /** Disable the `@` tag-insertion menu (existing tags still render). Defaults off — the file editor keeps tagging. */ disableTagging?: boolean /** - * Opt this surface into live collaborative editing. Set only by the Files page — - * the dedicated editing surface, which never streams agent output. The agent/Chat - * surface leaves it off, so collaboration and agent-streaming are disjoint by - * construction (they cannot both drive one editor and corrupt the shared doc). + * Opt this surface into live collaborative editing (Files page + the embedded chat file preview). + * Collaboration can coexist with agent streaming: while streaming, the growing content is applied to + * the shared Y.Doc as minimal CRDT diffs (see {@link applyStreamedMarkdownToLiveDoc}) rather than a + * full-document `setContent`, so the stream stays smooth and every peer sees it live. */ collaborative?: boolean /** @@ -257,9 +258,11 @@ export function LoadedRichMarkdownEditor({ /** * Collaboration is decided once at mount from synchronously-available inputs * (`settledRef` is set just above) via `useState`-init, and never changes — TipTap - * fixes the extension set at editor creation, so it cannot turn on later. Enabled - * only on a `collaborative` surface (the Files page, which never streams) for an - * editable, round-trip-safe, non-streaming workspace document with a known user. + * fixes the extension set at editor creation, so it cannot turn on later. Enabled on a + * `collaborative` surface (the Files page or the embedded chat file preview) for an editable, + * round-trip-safe workspace document with a known user, as long as it is not ALREADY streaming at + * mount (`!streamingAtMountRef.current`). An agent stream that begins AFTER mount is applied as CRDT + * diffs into the live doc, so collaboration and streaming coexist (see the streaming effect below). */ const [collaborationEnabled] = useState( () => @@ -751,9 +754,10 @@ export function LoadedRichMarkdownEditor({ }, [collaboration, editor, onCollabReadyChange, setCollabReady]) /** - * Owns editability for the collaborative lifecycle: `useEditor`'s `editable` is only - * the initial value, and the streaming/settle effect stays inert in collab mode — so - * re-apply here whenever collaboration readiness (synced + seeded) flips `isEditable`. + * Owns editability for the collaborative lifecycle: `useEditor`'s `editable` is only the initial + * value, and the streaming/settle effect only moves content in collab mode (never toggles + * editability) — so re-apply here whenever collaboration readiness (synced + seeded) or an agent + * stream flips `isEditable`. */ useEffect(() => { if (!editor || !collaborationEnabled) return @@ -800,13 +804,6 @@ export function LoadedRichMarkdownEditor({ const pendingCollapseRef = useRef(false) useEffect(() => { if (!editor) return - // Collaboration and agent-streaming are disjoint surfaces: collaboration is enabled - // only on the Files page, which never streams agent output, so in collab mode this - // reconcile loop stays fully inert. It must not run even defensively — its - // `setContent` would sync a full-document replace into the shared Y.Doc (the - // ySyncPlugin writes it regardless of `emitUpdate: false`), wiping peers' edits. - // Editability is owned by the reactive effect above. - if (collaborationEnabled) return const syncEditorBody = (body: string) => { if (body === lastSyncedBodyRef.current) return lastSyncedBodyRef.current = body @@ -831,6 +828,80 @@ export function LoadedRichMarkdownEditor({ mutate() }) } + // Collaborative surface: stream by applying a minimal CRDT diff into the live Y.Doc each frame + // (never `setContent`, which would replace the shared doc and wipe peers). Each diff renders + // locally and broadcasts to every peer, so the stream is smooth here and on other clients (e.g. + // the standalone Files page) alike. This branch moves content only — editability for the collab + // lifecycle is owned by the reactive effect above. + if (collaborationEnabled) { + if (isStreaming) { + wasStreamingRef.current = true + // Apply streamed diffs only after the shared doc has SEEDED (synced + seed flag, i.e. + // `collabReady`). Applying onto an unseeded (empty) doc would let the later seed CRDT-merge into + // it — transient garble. In the common case the doc is long seeded before an agent edit begins; + // in the rare stream-before-seed race we wait, and since `collabReady` is an effect dep this + // re-runs and applies once it lands (the read-only placeholder shows the base content meanwhile — + // see `showPlaceholder`). + if (!collabReady) return + const body = splitFrontmatter(content).body + if (body === lastSyncedBodyRef.current) return + pendingStreamBodyRef.current = body + if (streamRafRef.current !== null) return + const tick = () => { + const pending = pendingStreamBodyRef.current + if (pending === null || pending === lastSyncedBodyRef.current) { + streamRafRef.current = null + return + } + const shownBody = lastSyncedBodyRef.current + const extendsShown = shownBody === null || pending.startsWith(shownBody) + if (!streamIsIncrementalRef.current && !extendsShown) { + streamRafRef.current = null + return + } + if ( + pending.length > STREAM_REPARSE_THROTTLE_THRESHOLD && + performance.now() - lastStreamParseAtRef.current < STREAM_REPARSE_THROTTLE_MS + ) { + streamRafRef.current = requestAnimationFrame(tick) + return + } + const el = containerRef.current + const pinnedToBottom = el ? el.scrollHeight - el.scrollTop - el.clientHeight < 80 : false + // Defensive: a ready collab editor always has a ySync binding, so this applies; if one is + // somehow absent, bail this frame without advancing rather than looping. + if (!applyStreamedMarkdownToLiveDoc(editor, pending)) { + streamRafRef.current = null + return + } + streamRafRef.current = null + lastSyncedBodyRef.current = pending + lastStreamParseAtRef.current = performance.now() + if (!disableStreamingAutoScroll && el && pinnedToBottom) el.scrollTop = el.scrollHeight + } + streamRafRef.current = requestAnimationFrame(tick) + return + } + if (streamRafRef.current !== null) { + cancelAnimationFrame(streamRafRef.current) + streamRafRef.current = null + } + // Settle: once seeded, apply the final body once (the last streaming frame may have been throttled) + // so the Y.Doc exactly equals the streamed result; the durable server write then lands as a noop + // diff. If the seed has not arrived, keep `wasStreamingRef` set so the post-seed re-run applies. + if (wasStreamingRef.current && collabReady) { + wasStreamingRef.current = false + const finalBody = splitFrontmatter(content).body + if (finalBody !== lastSyncedBodyRef.current) { + runOffRender(() => { + if (applyStreamedMarkdownToLiveDoc(editor, finalBody)) { + lastSyncedBodyRef.current = finalBody + } + }) + } + } + return + } if (isStreaming) { wasStreamingRef.current = true if (editor.isEditable) { @@ -935,10 +1006,11 @@ export function LoadedRichMarkdownEditor({ [] ) - // Show the read-only placeholder only for a plain cold open — never during an agent stream. A stream - // that begins before the doc has seeded fills the (hidden) editor via Yjs, so gating the placeholder - // off while streaming lets that live content show through instead of hiding it behind stale markdown. - const showPlaceholder = collaborationEnabled && !collabReady && !isStreaming + // Show the read-only placeholder (the already-fetched markdown) whenever a collaborative doc has not yet + // seeded — including during an agent stream that begins before the seed lands. Streamed diffs are held + // until `collabReady` (see the streaming effect), so before then the editor is empty; the placeholder + // shows the base content until the seed swaps it in, avoiding both a blank frame and a garbled merge. + const showPlaceholder = collaborationEnabled && !collabReady return (
({ - mergeEditIntoLiveFileDocMock: - vi.fn< - ( - fileId: string, - markdown: string, - order?: { version?: number; baseVersion?: number } - ) => Promise - >(), - isLiveDocMergeInFlightMock: vi.fn<(fileId: string) => boolean>(), -})) - const { peekFileIntentMock } = vi.hoisted(() => ({ peekFileIntentMock: vi.fn(), })) -vi.mock('@/lib/realtime/notify', () => ({ - mergeEditIntoLiveFileDoc: mergeEditIntoLiveFileDocMock, - isLiveDocMergeInFlight: isLiveDocMergeInFlightMock, -})) - vi.mock('@/lib/copilot/tools/server/files/file-intent-store', () => ({ peekFileIntent: peekFileIntentMock, })) @@ -46,7 +29,6 @@ import type { ActiveFileIntent, ExecutionContext, StreamEvent } from '@/lib/copi const STREAM_ID = 'stream-1' const EDIT_TOOL_CALL_ID = 'edit-content-1' const WORKSPACE_FILE_TOOL_CALL_ID = 'workspace-file-1' -/** The durable version (`contentUpdatedAt`, epoch ms) the streamed base content is at. */ const BASE_VERSION_MS = 900_000 /** One args_delta chunk of the streamed `edit_content` JSON, as a driveable StreamEvent. */ @@ -91,9 +73,14 @@ const flushMicrotasks = async () => { await Promise.resolve() } -describe('processFilePreviewStreamEvent — live-doc streaming merge', () => { +/** + * The copilot preview adapter no longer merges the growing content into the file's live collaborative + * Y.Doc server-side — that is done client-side by the open editor as minimal CRDT diffs (see + * `applyStreamedMarkdownToLiveDoc`). These tests guard the surviving contract: the adapter still emits + * the growing `file_preview_content` events that drive the chat's inline preview. + */ +describe('processFilePreviewStreamEvent — preview content emission', () => { let state: FilePreviewAdapterState - let nowMs: number const execContext: ExecutionContext = { userId: 'user-1', workflowId: 'workflow-1', @@ -101,162 +88,55 @@ describe('processFilePreviewStreamEvent — live-doc streaming merge', () => { chatId: 'chat-1', messageId: 'msg-1', } + const events: Array<{ payload: Record }> = [] beforeEach(() => { vi.clearAllMocks() - mergeEditIntoLiveFileDocMock.mockResolvedValue(undefined) - isLiveDocMergeInFlightMock.mockReturnValue(false) - // Default: an append/patch base is available (a non-empty file) at durable version BASE_VERSION_MS, - // so the base-present gate passes and the streaming merge carries that base version. + events.length = 0 + // An append base is available (a non-empty file) at durable version BASE_VERSION_MS, so the preview + // text is composed as base + streamed content. peekFileIntentMock.mockResolvedValue({ existingContent: 'Base.', fileRecord: { contentUpdatedAt: new Date(BASE_VERSION_MS) }, }) state = createFilePreviewAdapterState() - nowMs = 1_000_000 - vi.spyOn(Date, 'now').mockImplementation(() => nowMs) - }) - - afterEach(() => { - vi.restoreAllMocks() }) async function drive(streamEvent: StreamEvent, intent: ActiveFileIntent) { const context = createStreamingContext() - // channelId resolves to '' when the event carries no scope. context.activeFileIntents.set('', intent) await processFilePreviewStreamEvent({ streamId: STREAM_ID, streamEvent, context, execContext, - options: { onEvent: vi.fn() }, + options: { + onEvent: (event) => { + events.push(event as { payload: Record }) + }, + }, state, }) } - it('merges the growing full content (base version, no durable version) into the live doc as it streams', async () => { - const intent = makeIntent({ operation: 'append', fileId: 'file-grow', fileName: 'notes.md' }) - - await drive(editContentDelta('{"content":"Hello'), intent) - await flushMicrotasks() - - // Advance past the throttle window so the next delta is due for another merge. - nowMs += 300 - await drive(editContentDelta(' world'), intent) - await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).toHaveBeenCalledTimes(2) - const [first, second] = mergeEditIntoLiveFileDocMock.mock.calls - // A full-file snapshot (base + streamed), never a diff; it grows across deltas. Each streaming merge - // carries `baseVersion` (the durable version it was built from) to order it — never `version`, which - // rides the final edit_content write; so the relay drops it if a newer durable write has since landed - // but never records it as a durable checkpoint. - expect(first[0]).toBe('file-grow') - expect(first[1]).toContain('Base.') - expect(first[1]).toContain('Hello') - expect(first[2]?.baseVersion).toBe(BASE_VERSION_MS) - expect(first[2]?.version).toBeUndefined() - expect(second[1]).toContain('Hello world') - expect(second[1].length).toBeGreaterThan(first[1].length) - expect(second[2]?.baseVersion).toBe(BASE_VERSION_MS) - }) - - it('falls back to updatedAt for baseVersion when the file has no content version', async () => { - // A legacy file with no `contentUpdatedAt` — the base version must fall back to `updatedAt`, the SAME - // line the relay's synced version is on, so the snapshot is still ordered (not shipped unordered). - const UPDATED_AT_MS = 850_000 - peekFileIntentMock.mockResolvedValue({ - existingContent: 'Base.', - fileRecord: { contentUpdatedAt: null, updatedAt: new Date(UPDATED_AT_MS) }, - }) - const intent = makeIntent({ operation: 'append', fileId: 'file-legacy', fileName: 'notes.md' }) - - await drive(editContentDelta('{"content":"Hello'), intent) - await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).toHaveBeenCalledTimes(1) - expect(mergeEditIntoLiveFileDocMock.mock.calls[0][2]?.baseVersion).toBe(UPDATED_AT_MS) - }) - - it('throttles merges: two deltas within LIVE_DOC_MERGE_THROTTLE_MS yield one merge', async () => { - const intent = makeIntent({ - operation: 'append', - fileId: 'file-throttle', - fileName: 'notes.md', - }) - - await drive(editContentDelta('{"content":"Hel'), intent) - await flushMicrotasks() - - // 100ms < 250ms throttle → the second snapshot is dropped, not merged. - nowMs += 100 - await drive(editContentDelta('lo world'), intent) - await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).toHaveBeenCalledTimes(1) - }) - - it('does not merge for a non-markdown file (no collaborative room)', async () => { - const intent = makeIntent({ operation: 'append', fileId: 'file-txt', fileName: 'notes.txt' }) - - await drive(editContentDelta('{"content":"plain text body'), intent) - await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).not.toHaveBeenCalled() - }) - - it('does not merge when base content loads without a version (unordered-wipe guard)', async () => { - // Base text is available but the intent carries no file record → no baseVersion. The relay would - // treat a versionless snapshot as unordered (never stale), so it must be skipped fail-closed. - peekFileIntentMock.mockResolvedValue({ existingContent: 'Base.' }) - const intent = makeIntent({ operation: 'append', fileId: 'file-nover', fileName: 'notes.md' }) - - await drive(editContentDelta('{"content":"Hello'), intent) - await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).not.toHaveBeenCalled() - }) - - it('does not merge an append before base content loads (base-less-wipe guard)', async () => { - // No pending intent base is available yet → session.baseContent stays undefined. - peekFileIntentMock.mockResolvedValue(undefined) - const intent = makeIntent({ operation: 'append', fileId: 'file-append', fileName: 'notes.md' }) - - await drive(editContentDelta('{"content":"\\n- appended line'), intent) - await flushMicrotasks() - - // A base-less snapshot would diff to a delete-everything wipe of the seeded doc, so it must be skipped. - expect(mergeEditIntoLiveFileDocMock).not.toHaveBeenCalled() - }) - - it('does not stream an update (from-scratch rewrite) — it would blank the doc mid-stream', async () => { - const intent = makeIntent({ operation: 'update', fileId: 'file-update', fileName: 'notes.md' }) - - await drive(editContentDelta('{"content":"Rewritten intro'), intent) - await flushMicrotasks() - - // Update streams a partial rewrite; diffing the full doc toward it would delete most of the file - // until it grows back, so update applies atomically at the final durable write instead. - expect(mergeEditIntoLiveFileDocMock).not.toHaveBeenCalled() - }) + function previewContent(): string { + return events + .filter((e) => e.payload?.previewPhase === 'file_preview_content') + .map((e) => String(e.payload?.content ?? '')) + .join('') + } - it('skips the merge while one is already in flight for the file (does not backlog / advance throttle)', async () => { - isLiveDocMergeInFlightMock.mockReturnValue(true) - const intent = makeIntent({ operation: 'append', fileId: 'file-busy', fileName: 'notes.md' }) + it('emits the growing preview content (base + streamed) for an append stream', async () => { + const intent = makeIntent({ operation: 'append', fileId: 'file-grow', fileName: 'notes.md' }) await drive(editContentDelta('{"content":"Hello'), intent) await flushMicrotasks() - - expect(mergeEditIntoLiveFileDocMock).not.toHaveBeenCalled() - - // The dropped in-flight tick must NOT advance the throttle window, so once the in-flight merge - // clears the very next delta merges immediately — no wait for a fresh throttle interval. - isLiveDocMergeInFlightMock.mockReturnValue(false) await drive(editContentDelta(' world'), intent) await flushMicrotasks() - expect(mergeEditIntoLiveFileDocMock).toHaveBeenCalledTimes(1) - expect(mergeEditIntoLiveFileDocMock.mock.calls[0][0]).toBe('file-busy') + const combined = previewContent() + expect(combined).toContain('Base.') + expect(combined).toContain('Hello') + expect(combined).toContain('world') }) }) diff --git a/apps/sim/lib/copilot/request/go/file-preview-adapter.ts b/apps/sim/lib/copilot/request/go/file-preview-adapter.ts index ccc911bd08e..3d16cff4bb1 100644 --- a/apps/sim/lib/copilot/request/go/file-preview-adapter.ts +++ b/apps/sim/lib/copilot/request/go/file-preview-adapter.ts @@ -25,9 +25,7 @@ import { loadWorkspaceFileTextForPreview, type WorkspaceFilePreviewBase, } from '@/lib/copilot/tools/server/files/file-preview' -import { isLiveDocMergeInFlight, mergeEditIntoLiveFileDoc } from '@/lib/realtime/notify' import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' const logger = createLogger('CopilotFilePreviewAdapter') @@ -43,8 +41,6 @@ type FilePreviewStreamState = { session: FilePreviewSession lastEmittedPreviewText: string lastSnapshotAt: number - /** Epoch ms of the last merge of the growing content into the file's live collaborative Y.Doc. */ - lastLiveMergeAt: number } type ParsedWorkspaceFileArgs = { @@ -57,12 +53,6 @@ type ParsedWorkspaceFileArgs = { const PATCH_PREVIEW_SNAPSHOT_INTERVAL_MS = 80 const DELTA_PREVIEW_CHECKPOINT_INTERVAL_MS = 1000 -/** - * Throttle for merging the growing copilot content into the file's live collaborative Y.Doc as it - * streams. ~4 merges/sec reads as live while keeping CRDT diff churn and relay load bounded - * regardless of token rate; the final durable `edit_content` write is the stream-end flush. - */ -const LIVE_DOC_MERGE_THROTTLE_MS = 250 function asJsonRecord(value: unknown): JsonRecord | undefined { return value && typeof value === 'object' && !Array.isArray(value) @@ -273,7 +263,6 @@ function buildPreviewSessionFromIntent( operation: intent.operation, ...(intent.edit ? { edit: intent.edit } : {}), ...(typeof current?.baseContent === 'string' ? { baseContent: current.baseContent } : {}), - ...(typeof current?.baseVersion === 'number' ? { baseVersion: current.baseVersion } : {}), previewText: current?.previewText ?? '', previewVersion: current?.previewVersion ?? 0, status: current?.status ?? 'pending', @@ -412,16 +401,12 @@ export async function processFilePreviewStreamEvent(input: { session = { ...session, baseContent: previewBase.text, - ...(previewBase.baseVersion !== undefined - ? { baseVersion: previewBase.baseVersion } - : {}), } } filePreviewState.set(toolCallId, { session, lastEmittedPreviewText: '', lastSnapshotAt: 0, - lastLiveMergeAt: 0, }) await persistFilePreviewSession(session) @@ -487,16 +472,12 @@ export async function processFilePreviewStreamEvent(input: { session = { ...session, baseContent: previewBase.text, - ...(previewBase.baseVersion !== undefined - ? { baseVersion: previewBase.baseVersion } - : {}), } } filePreviewState.set(intent.toolCallId, { session, lastEmittedPreviewText: '', lastSnapshotAt: 0, - lastLiveMergeAt: 0, }) await persistFilePreviewSession(session) @@ -566,7 +547,6 @@ export async function processFilePreviewStreamEvent(input: { session: nextSession, lastEmittedPreviewText: previewText, lastSnapshotAt: Date.now(), - lastLiveMergeAt: 0, }) await persistFilePreviewSession(nextSession) await emitPreviewEvent(streamEvent, options, { @@ -600,7 +580,6 @@ export async function processFilePreviewStreamEvent(input: { session: buildPreviewSessionFromIntent(streamId, editIntent), lastEmittedPreviewText: '', lastSnapshotAt: 0, - lastLiveMergeAt: 0, } if ( @@ -619,15 +598,9 @@ export async function processFilePreviewStreamEvent(input: { } ) if (typeof intentBase?.existingContent === 'string') { - // Same version line as the seed/persist (`contentUpdatedAt ?? updatedAt`), so the stream's - // base is comparable to the relay's synced version even when the file has no content version. - const baseVersion = ( - intentBase.fileRecord?.contentUpdatedAt ?? intentBase.fileRecord?.updatedAt - )?.getTime() const seededSession: FilePreviewSession = { ...currentPreview.session, baseContent: intentBase.existingContent, - ...(baseVersion !== undefined ? { baseVersion } : {}), ...(intentBase.edit ? { edit: intentBase.edit } : {}), } currentPreview = { @@ -665,41 +638,12 @@ export async function processFilePreviewStreamEvent(input: { await persistFilePreviewSession(nextSession) - // Stream the growing content into the file's LIVE collaborative Y.Doc (when a room is open) - // so collaborators watching the file see the copilot write stream in via Yjs — the AI as a - // CRDT peer, applied by the relay as a minimal `updateYFragment` diff. Fire-and-forget so a - // slow relay never stalls the stream. Pass `baseVersion` (the durable version this snapshot is - // built from) so the relay drops it if a NEWER durable write has landed since — e.g. a - // concurrent human save — rather than diffing the live doc back toward stale content and - // clobbering that edit (which a later persist would then write over the durable file). It is - // never recorded as a checkpoint; the final `edit_content` write carries the real version and - // reconciles the durable file. No-op for `create` (never streams here) and for a file with no - // open room (the relay reports `applied: false`). - // - // Gates: markdown only (non-markdown has no collaborative room). Only `append`/`patch` stream - // — they build on the existing content, so they need the base loaded (a base-less snapshot - // would diff to a delete-everything wipe of the seeded doc). `update` is a from-scratch - // rewrite: streaming its partial content would diff the full doc toward a fragment and blank - // it mid-stream, so it applies atomically at the final durable write instead. Skip while a - // merge is in flight for this file — one at a time, and don't advance the throttle on a - // no-op — so a slow relay can't backlog stale snapshots or make the doc lag the stream. - // Require a numeric `baseVersion`: without it the relay can't order the snapshot and would - // treat it as unordered (never stale), so a rare base with no version (no file record) is - // fail-closed — skip the live merge rather than risk clobbering a concurrent durable write. - const dueForLiveMerge = - nextSession.fileId !== undefined && - isMarkdownFile({ type: editIntent.contentType, name: nextSession.fileName ?? '' }) && - (editIntent.operation === 'append' || editIntent.operation === 'patch') && - currentPreview.session.baseContent !== undefined && - nextSession.baseVersion !== undefined && - !isLiveDocMergeInFlight(nextSession.fileId) && - now - currentPreview.lastLiveMergeAt >= LIVE_DOC_MERGE_THROTTLE_MS - const nextLiveMergeAt = dueForLiveMerge ? now : currentPreview.lastLiveMergeAt - if (dueForLiveMerge && nextSession.fileId) { - void mergeEditIntoLiveFileDoc(nextSession.fileId, nextSession.previewText, { - baseVersion: nextSession.baseVersion, - }) - } + // The growing content is NOT merged into the live collaborative Y.Doc from here. When a + // collaborative editor for this file is open, that client applies the stream to the shared + // doc as minimal CRDT diffs (see `applyStreamedMarkdownToLiveDoc` in the editor), which + // renders smoothly locally AND broadcasts to every peer — so a server-side streaming merge + // would double-write the shared doc. The final `edit_content` durable write still reconciles + // the file and seeds any late joiner. if ( nextSession.operation === 'patch' && @@ -709,7 +653,6 @@ export async function processFilePreviewStreamEvent(input: { session: nextSession, lastEmittedPreviewText: currentPreview.lastEmittedPreviewText, lastSnapshotAt: currentPreview.lastSnapshotAt, - lastLiveMergeAt: nextLiveMergeAt, }) } else { const previewUpdate = buildPreviewContentUpdate( @@ -724,7 +667,6 @@ export async function processFilePreviewStreamEvent(input: { session: nextSession, lastEmittedPreviewText: nextSession.previewText, lastSnapshotAt: previewUpdate.lastSnapshotAt, - lastLiveMergeAt: nextLiveMergeAt, }) await emitPreviewEvent(streamEvent, options, { @@ -746,7 +688,6 @@ export async function processFilePreviewStreamEvent(input: { session: currentPreview.session, lastEmittedPreviewText: currentPreview.lastEmittedPreviewText, lastSnapshotAt: currentPreview.lastSnapshotAt, - lastLiveMergeAt: currentPreview.lastLiveMergeAt, }) } } @@ -780,7 +721,6 @@ export async function processFilePreviewStreamEvent(input: { session: currentPreview.session, lastEmittedPreviewText: currentPreview.session.previewText, lastSnapshotAt: Date.now(), - lastLiveMergeAt: currentPreview.lastLiveMergeAt, }) await emitPreviewEvent(streamEvent, options, { toolCallId: currentPreview.session.toolCallId, @@ -812,7 +752,6 @@ export async function processFilePreviewStreamEvent(input: { session: completedSession, lastEmittedPreviewText: completedSession.previewText, lastSnapshotAt: Date.now(), - lastLiveMergeAt: currentPreview.lastLiveMergeAt, }) await persistFilePreviewSession(completedSession) } diff --git a/apps/sim/lib/copilot/request/session/file-preview-session-contract.ts b/apps/sim/lib/copilot/request/session/file-preview-session-contract.ts index f29624f1562..a2e96208ba1 100644 --- a/apps/sim/lib/copilot/request/session/file-preview-session-contract.ts +++ b/apps/sim/lib/copilot/request/session/file-preview-session-contract.ts @@ -16,10 +16,6 @@ export interface FilePreviewSession { operation?: string edit?: Record baseContent?: string - /** The durable version (`contentUpdatedAt`, epoch ms) `baseContent` is at — the stream's causal base, - * passed to the relay so a snapshot is dropped if a newer durable write landed. Undefined for a - * legacy file with no recorded version, or a session with no loaded base. */ - baseVersion?: number previewText: string previewVersion: number updatedAt: string diff --git a/apps/sim/lib/copilot/request/session/file-preview-session.ts b/apps/sim/lib/copilot/request/session/file-preview-session.ts index c907ef020a1..df93b3a03ac 100644 --- a/apps/sim/lib/copilot/request/session/file-preview-session.ts +++ b/apps/sim/lib/copilot/request/session/file-preview-session.ts @@ -78,7 +78,6 @@ export function createFilePreviewSession(input: { operation?: string edit?: Record baseContent?: string - baseVersion?: number previewText?: string previewVersion?: number status?: FilePreviewStatus @@ -97,7 +96,6 @@ export function createFilePreviewSession(input: { ...(input.operation ? { operation: input.operation } : {}), ...(input.edit ? { edit: input.edit } : {}), ...(typeof input.baseContent === 'string' ? { baseContent: input.baseContent } : {}), - ...(typeof input.baseVersion === 'number' ? { baseVersion: input.baseVersion } : {}), previewText: input.previewText ?? '', previewVersion: input.previewVersion ?? 0, updatedAt: input.updatedAt ?? new Date().toISOString(), diff --git a/apps/sim/lib/copilot/tools/server/files/file-preview.ts b/apps/sim/lib/copilot/tools/server/files/file-preview.ts index 698d524d463..4ea1919395e 100644 --- a/apps/sim/lib/copilot/tools/server/files/file-preview.ts +++ b/apps/sim/lib/copilot/tools/server/files/file-preview.ts @@ -141,16 +141,9 @@ function buildAppendPreview(existingContent: string, incomingContent: string): s * before Redis holds `existingContent`, which would make append previews look like * full-file replacement until the intent landed. */ -/** - * The base content a copilot edit is computed against, plus the durable version (epoch ms) that content - * is at. The version is the stream's causal base: the relay drops a streaming snapshot if a NEWER durable - * write landed than this, so a concurrent human edit is never clobbered. Derived as - * `contentUpdatedAt ?? updatedAt` — the SAME version line the seed/persist use — so it is directly - * comparable to the relay's recorded synced version. - */ +/** The base content a copilot append/patch edit is computed against, to compose the streamed preview. */ export interface WorkspaceFilePreviewBase { text: string - baseVersion: number } export async function loadWorkspaceFileTextForPreview( @@ -163,7 +156,6 @@ export async function loadWorkspaceFileTextForPreview( const buffer = await fetchWorkspaceFileBuffer(record) return { text: buffer.toString('utf-8'), - baseVersion: (record.contentUpdatedAt ?? record.updatedAt).getTime(), } } catch (error) { logger.warn('Failed to load workspace file text for preview', { diff --git a/apps/sim/lib/realtime/notify.test.ts b/apps/sim/lib/realtime/notify.test.ts index 20689dd745d..98196492257 100644 --- a/apps/sim/lib/realtime/notify.test.ts +++ b/apps/sim/lib/realtime/notify.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/utils/urls', () => ({ getSocketServerUrl: () => 'http://realtime' })) vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) -import { isLiveDocMergeInFlight, mergeEditIntoLiveFileDoc } from './notify' +import { mergeEditIntoLiveFileDoc } from './notify' describe('mergeEditIntoLiveFileDoc', () => { afterEach(() => { @@ -24,25 +24,12 @@ describe('mergeEditIntoLiveFileDoc', () => { expect.objectContaining({ method: 'POST', headers: expect.objectContaining({ 'x-api-key': 'secret' }), - // A durable write sends `version`; the undefined `baseVersion` is dropped by JSON.stringify. + // A durable write sends `version`; an unversioned (legacy) call would drop it via JSON.stringify. body: JSON.stringify({ fileId: 'file-1', markdown: '# hello', version: 42 }), }) ) }) - it('sends baseVersion (not version) for a streaming snapshot', async () => { - const fetchMock = vi.fn().mockResolvedValue({ ok: true }) - vi.stubGlobal('fetch', fetchMock) - - await mergeEditIntoLiveFileDoc('file-1', '# hello', { baseVersion: 1234 }) - - // The relay orders the snapshot by its causal baseVersion without recording it — durable version - // stays absent on the wire. - expect(fetchMock.mock.calls[0][1].body).toBe( - JSON.stringify({ fileId: 'file-1', markdown: '# hello', baseVersion: 1234 }) - ) - }) - it('never throws when the realtime call fails (best-effort)', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) await expect( @@ -57,61 +44,42 @@ describe('mergeEditIntoLiveFileDoc', () => { ).resolves.toBeUndefined() }) - it('reports isLiveDocMergeInFlight while a merge runs and clears when it settles', async () => { - let resolveFetch: (value: { ok: boolean }) => void = () => {} - vi.stubGlobal( - 'fetch', - vi.fn(() => new Promise((resolve) => (resolveFetch = resolve))) - ) - - expect(isLiveDocMergeInFlight('file-flight')).toBe(false) - const run = mergeEditIntoLiveFileDoc('file-flight', 'v1') - await Promise.resolve() - // The streaming caller checks this to skip a redundant merge (and not advance its throttle) while - // one is in flight, so a slow relay can't backlog stale snapshots. - expect(isLiveDocMergeInFlight('file-flight')).toBe(true) - - resolveFetch({ ok: true }) - await run - expect(isLiveDocMergeInFlight('file-flight')).toBe(false) - }) - - it('a durable (versioned) merge waits for an in-flight streaming merge, then applies last', async () => { - let resolveStream: (value: { ok: boolean }) => void = () => {} + it('a later durable merge waits for an in-flight earlier one, then applies last', async () => { + let resolveFirst: (value: { ok: boolean }) => void = () => {} const fetchMock = vi .fn() - .mockImplementationOnce(() => new Promise((resolve) => (resolveStream = resolve))) + .mockImplementationOnce(() => new Promise((resolve) => (resolveFirst = resolve))) .mockResolvedValue({ ok: true }) vi.stubGlobal('fetch', fetchMock) - const stream = mergeEditIntoLiveFileDoc('file-durable', 'partial') // versionless, in flight + const first = mergeEditIntoLiveFileDoc('file-durable', 'earlier', { version: 99 }) // in flight await Promise.resolve() - const durable = mergeEditIntoLiveFileDoc('file-durable', 'final content', { version: 100 }) // versioned + const durable = mergeEditIntoLiveFileDoc('file-durable', 'final content', { version: 100 }) await Promise.resolve() await Promise.resolve() - // The durable write waits for the in-flight streaming merge → its fetch has not fired yet, so it - // cannot be reordered before a straggler and cannot be clobbered by one. + // The later write waits for the in-flight earlier one → its fetch has not fired yet, so it cannot be + // reordered before a straggler and cannot be clobbered by one. expect(fetchMock).toHaveBeenCalledTimes(1) - resolveStream({ ok: true }) - await stream + resolveFirst({ ok: true }) + await first await durable - // Only after the streaming merge completed does the durable (final) merge apply — always last. + // Only after the earlier merge completed does the later (final) merge apply — always last. expect(fetchMock).toHaveBeenCalledTimes(2) expect(fetchMock.mock.calls[1][1].body).toBe( JSON.stringify({ fileId: 'file-durable', markdown: 'final content', version: 100 }) ) }) - it('serializes concurrent durable writes behind a streaming merge, strictly in order', async () => { - const applied: Array = [] + it('serializes concurrent durable writes to a file strictly in order', async () => { + const applied: number[] = [] const resolvers: Array<() => void> = [] vi.stubGlobal( 'fetch', vi.fn((_url: string, init: { body: string }) => { - applied.push(JSON.parse(init.body).version ?? 'stream') + applied.push(JSON.parse(init.body).version) return new Promise<{ ok: boolean }>((resolve) => resolvers.push(() => resolve({ ok: true })) ) @@ -121,22 +89,22 @@ describe('mergeEditIntoLiveFileDoc', () => { for (let i = 0; i < 6; i++) await Promise.resolve() } - const s = mergeEditIntoLiveFileDoc('file-order', 's') // streaming, in flight + const s = mergeEditIntoLiveFileDoc('file-order', 's', { version: 0 }) // in flight await flush() - // Two durable writes arrive while the streaming merge is in flight — both must chain, not both + // Two later durable writes arrive while the first merge is in flight — both must chain, not both // resume-and-fire concurrently. const a = mergeEditIntoLiveFileDoc('file-order', 'a', { version: 1 }) const b = mergeEditIntoLiveFileDoc('file-order', 'b', { version: 2 }) await flush() - expect(applied).toEqual(['stream']) // A and B queued behind streaming + expect(applied).toEqual([0]) // A and B queued behind the in-flight first merge - resolvers[0]() // finish streaming → A applies next (not B) + resolvers[0]() // finish first → A applies next (not B) await flush() - expect(applied).toEqual(['stream', 1]) + expect(applied).toEqual([0, 1]) resolvers[1]() // finish A → B applies after A await flush() - expect(applied).toEqual(['stream', 1, 2]) + expect(applied).toEqual([0, 1, 2]) resolvers[2]() await Promise.all([s, a, b]) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index a67f86401c4..b377635c49b 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -110,49 +110,39 @@ export async function notifyFolderResourceChanged( } /** - * How a live-doc merge is positioned on the file's monotonic version line. Pass one key or the other, - * never both; passing neither applies the merge without ordering it (legacy). + * How a durable live-doc merge is positioned on the file's monotonic version line. Omit `version` to + * apply the merge without ordering it (legacy). */ export interface LiveFileDocMergeOrder { /** A durable write's `contentUpdatedAt` (epoch ms): applied only if newer than the version the doc - * already incorporates, AND recorded as the synced version. */ + * already incorporates, AND recorded as the synced version (the persist If-Match guard). */ version?: number - /** A streaming snapshot's causal base — the durable `contentUpdatedAt` it was built from. The relay - * drops the snapshot if a NEWER durable version was recorded than this (a concurrent write landed), and - * never records it as a checkpoint. */ - baseVersion?: number } /** - * Best-effort: ask the realtime relay to merge a copilot edit into a file's LIVE collaborative - * document, so open editors see it stream in as a CRDT merge (Stage C) rather than the file changing - * underneath them. No-op when no doc is (or was recently) live (the relay reports `applied: false`). - * The file itself is written durably by the caller regardless — this only drives the live view. - * Never throws. + * Best-effort: ask the realtime relay to merge a durable copilot/file write into a file's LIVE + * collaborative document, so open editors reconcile to it as a CRDT merge rather than the file changing + * underneath them, and a late joiner is seeded from it. No-op when no doc is (or was recently) live (the + * relay reports `applied: false`). The file itself is written durably by the caller regardless — this + * only drives the live view. Never throws. * - * The former clobber gap — an open editor's autosave dropping this edit — is now closed: a - * collaborative editor no longer client-autosaves (the relay persists the shared doc to markdown - * server-side), and the relay applies this merge THROUGH the shared Redis stream, so it reaches the - * live doc on whichever task holds it and can't go stale relative to this direct write. + * (Streaming copilot output is NOT merged here: the open editor applies the stream client-side as minimal + * CRDT diffs — see `applyStreamedMarkdownToLiveDoc` — which renders smoothly and broadcasts to peers. This + * merge is the stream-end durable reconcile, and by then it is usually a noop diff.) * - * A durable caller awaits this (so the fetch dispatches before the route handler returns); the copilot - * streaming caller fires and forgets it. Bounded to {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency - * only when the socket pod is unreachable. + * The former clobber gap — an open editor's autosave dropping this edit — is closed: a collaborative + * editor no longer client-autosaves (the relay persists the shared doc to markdown server-side), and the + * relay applies this merge THROUGH the shared Redis stream, so it reaches the live doc on whichever task + * holds it and can't go stale relative to this direct write. * - * `order` ({@link LiveFileDocMergeOrder}) positions this merge so a stale write never regresses the doc. - * A durable `version` applies only if newer than the version the doc already incorporates, and is recorded - * as the synced version (the persist If-Match guard). A streaming `baseVersion` is the durable version the - * snapshot was built from: the relay drops the snapshot if a NEWER durable write has since landed — so a - * concurrent human edit is never clobbered (nor later persisted over the file) — and never records it, so - * the synced version stays pinned to the last durable write, which the copilot tool's final `edit_content` - * write carries, reconciling the durable file. + * The caller awaits this so the fetch dispatches before the route handler returns. Bounded to + * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. * - * Ordering is enforced at two scales. Within this process, merges for a file run on a single serialized - * chain — each chained after the current tail — so a durable write applies after any in-flight streaming - * merge and after every earlier durable write, never concurrently. Across processes, the per-process - * chain does not apply, so the relay orders merges by that monotonic version under a cluster-wide lock. - * The copilot streaming caller uses {@link isLiveDocMergeInFlight} to skip redundant snapshots while one - * is in flight, so a slow relay can't backlog stale snapshots. + * `order.version` positions the merge so a stale write never regresses the doc: it applies only if newer + * than the version the doc already incorporates, and is recorded as the synced version. Ordering is + * enforced at two scales: within this process, merges for a file run on a single serialized chain (each + * chained after the current tail) so writes never apply concurrently; across processes the relay orders + * by that monotonic version under a cluster-wide lock. */ export async function mergeEditIntoLiveFileDoc( fileId: string, @@ -173,16 +163,6 @@ export async function mergeEditIntoLiveFileDoc( * because {@link applyLiveFileDocMerge} never throws. Absent when the file's chain is idle. */ const liveDocMergeChain = new Map>() -/** - * Whether a live-doc merge is currently running or queued for the file. The copilot streaming caller - * checks this to skip a redundant snapshot (and to not advance its send throttle) while a merge is in - * flight — bounding the stream to one live merge per file at a time without backlogging stale - * snapshots behind a slow relay. - */ -export function isLiveDocMergeInFlight(fileId: string): boolean { - return liveDocMergeChain.has(fileId) -} - /** POST the merge to the relay. Never throws (a live-doc merge is best-effort). */ async function applyLiveFileDocMerge( fileId: string, @@ -194,13 +174,11 @@ async function applyLiveFileDocMerge( method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, // `version` (durable `contentUpdatedAt`) records the synced version the live doc now incorporates - // (the persist If-Match guard); `baseVersion` is a streaming snapshot's causal base, checked (drop - // if a newer durable landed) but never recorded. JSON.stringify drops whichever is undefined. + // (the persist If-Match guard). JSON.stringify drops it when undefined (an unordered legacy merge). body: JSON.stringify({ fileId, markdown, version: order.version, - baseVersion: order.baseVersion, }), signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), }) From 240e314212a43b408a078d0760e374e69d1959b2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 30 Jul 2026 22:59:50 -0700 Subject: [PATCH 02/15] fix(files): apply agent stream as a true CRDT peer + guard base-less snapshots Review round 1 (Greptile P1s): - apply the stream against a private shadow replica (seeded from the live doc at stream start) and relay only the agent's own delta into the shared doc, so a concurrent peer edit to a region the agent snapshot didn't include is no longer reverted (previously the whole-body reconcile deleted it) - gate append snapshots on "must extend the base": a base-less append fragment (emitted before the base loads) can no longer reconcile the seeded doc to a wipe; patch still legitimately replaces a mid-region - gate the apply on collabReady so diffs never land on an unseeded doc; keep the placeholder visible until the seed swaps in - plumb streamOperation through the preview surfaces to drive the append gate - add a peer-edit-preservation test (fails under whole-body reconcile) and refresh the undo-isolation + broadcast tests for the session API --- .../components/file-viewer/file-viewer.tsx | 3 + .../apply-streamed-markdown.test.ts | 73 ++++++++++-------- .../collaboration/apply-streamed-markdown.ts | 74 +++++++++++++++---- .../rich-markdown-editor.tsx | 53 ++++++++++--- .../resource-content/resource-content.tsx | 4 + 5 files changed, 152 insertions(+), 55 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index bba37374240..006012191ac 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -108,6 +108,7 @@ interface FileViewerProps { streamingContent?: string isAgentEditing?: boolean streamIsIncremental?: boolean + streamOperation?: string disableStreamingAutoScroll?: boolean previewContextKey?: string /** @@ -150,6 +151,7 @@ function FileViewerContent({ streamingContent, isAgentEditing, streamIsIncremental, + streamOperation, disableStreamingAutoScroll = false, previewContextKey, collaborative, @@ -196,6 +198,7 @@ function FileViewerContent({ streamingContent={streamingContent} isAgentEditing={isAgentEditing} streamIsIncremental={streamIsIncremental} + streamOperation={streamOperation} disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} collaborative={collaborative} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts index c9aa02f732f..493fea42334 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts @@ -6,7 +6,7 @@ import { afterEach, beforeAll, describe, expect, it } from 'vitest' import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' import { createMarkdownEditorExtensions } from '../editor-extensions' -import { applyStreamedMarkdownToLiveDoc } from './apply-streamed-markdown' +import { applyAgentStreamFrame, beginAgentStream, endAgentStream } from './apply-streamed-markdown' beforeAll(() => { // jsdom does not implement elementFromPoint; the Placeholder extension's viewport tracking calls it @@ -48,11 +48,13 @@ function track(t: { editor: Editor; doc: Y.Doc; awareness: Awareness }) { return t } -describe('applyStreamedMarkdownToLiveDoc', () => { +describe('agent-stream applier', () => { it('streams agent content into the live collaborative doc and broadcasts it as Yjs ops', () => { const { editor, doc } = track(makeCollabEditor()) - expect(applyStreamedMarkdownToLiveDoc(editor, '# Title\n\nHello world.')).toBe(true) + const session = beginAgentStream(editor) + expect(session).not.toBeNull() + expect(applyAgentStreamFrame(editor, session!, '# Title\n\nHello world.')).toBe(true) expect(editor.getText()).toContain('Hello world') // The write lands as ops on the shared doc, so any peer receives it (this is what makes a @@ -61,25 +63,27 @@ describe('applyStreamedMarkdownToLiveDoc', () => { Y.applyUpdate(peer, Y.encodeStateAsUpdate(doc)) expect(peer.getXmlFragment('default').toString()).toContain('Hello world') peer.destroy() + endAgentStream(session!) }) - it('returns false when the doc is not yet bound (no ySync binding)', () => { - // A plain editor with no collaboration has no ySync binding, so the applier reports "not ready" - // rather than throwing — the streaming tick re-arms until the doc is seeded. + it('beginAgentStream returns null when the editor has no ySync binding', () => { + // A plain editor with no collaboration has no ySync binding, so a stream cannot start against it. const editor = new Editor({ extensions: createMarkdownEditorExtensions({ placeholder: '' }), content: '', }) teardown.push(() => editor.destroy()) - expect(applyStreamedMarkdownToLiveDoc(editor, '# Nope')).toBe(false) + expect(beginAgentStream(editor)).toBeNull() }) it('keeps agent-streamed ops out of the undo stack while user edits stay undoable', () => { const { editor } = track(makeCollabEditor()) - applyStreamedMarkdownToLiveDoc(editor, '# Streamed\n\nAgent wrote this.') - // The streamed op used AGENT_STREAM_ORIGIN, which the Collaboration UndoManager does not track — - // so there is nothing to undo, and an undo must not revert the agent's content. + const session = beginAgentStream(editor)! + applyAgentStreamFrame(editor, session, '# Streamed\n\nAgent wrote this.') + endAgentStream(session) + // The streamed op relayed under a non-`ySyncPluginKey` origin, which the Collaboration UndoManager + // does not track — so there is nothing to undo, and an undo must not revert the agent's content. expect(editor.can().undo()).toBe(false) editor.commands.undo() expect(editor.getText()).toContain('Agent wrote this') @@ -95,31 +99,38 @@ describe('applyStreamedMarkdownToLiveDoc', () => { expect(editor.getText()).toContain('Agent wrote this') }) - it('merges an agent write with a concurrent peer edit (minimal diff, no clobber)', () => { + it('preserves a concurrent peer edit to a region the agent snapshot does not include', () => { + // This is the core "AI as a CRDT peer" guarantee: the agent relays only its OWN delta (computed + // against a private shadow), never a whole-document reconcile that would revert a peer's edit. const { editor, doc } = track(makeCollabEditor()) - applyStreamedMarkdownToLiveDoc(editor, 'Alpha paragraph.\n\nBeta paragraph.') - - // A peer forks the current state and edits the FIRST paragraph directly on the shared type… - const remote = new Y.Doc() - Y.applyUpdate(remote, Y.encodeStateAsUpdate(doc)) - const remoteFrag = remote.getXmlFragment('default') - remote.transact(() => { - const firstPara = remoteFrag.get(0) as Y.XmlElement + + const session = beginAgentStream(editor)! + applyAgentStreamFrame(editor, session, 'Alpha paragraph.\n\nBeta paragraph.') + + // A peer edits the FIRST paragraph directly on the shared doc — the agent's later snapshot still + // carries the ORIGINAL first paragraph (it was built from the base, before this edit). + const peer = new Y.Doc() + Y.applyUpdate(peer, Y.encodeStateAsUpdate(doc)) + const peerFrag = peer.getXmlFragment('default') + peer.transact(() => { + const firstPara = peerFrag.get(0) as Y.XmlElement const textNode = firstPara.get(0) as Y.XmlText textNode.insert(textNode.toString().length, ' EDITED') }) + Y.applyUpdate(doc, Y.encodeStateAsUpdate(peer, Y.encodeStateVector(doc))) + peer.destroy() - // …while the agent rewrites the SECOND paragraph through the live editor binding. - applyStreamedMarkdownToLiveDoc(editor, 'Alpha paragraph.\n\nBeta paragraph, expanded.') - - // Exchange updates both ways (as the relay would); a full-document replace would have clobbered - // the peer's concurrent edit — a minimal `updateYFragment` diff preserves both. - Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote, Y.encodeStateVector(doc))) - Y.applyUpdate(remote, Y.encodeStateAsUpdate(doc, Y.encodeStateVector(remote))) - - const merged = doc.getXmlFragment('default').toString() - expect(merged).toContain('EDITED') - expect(merged).toContain('expanded') - remote.destroy() + // The agent appends a third paragraph. Its snapshot's first paragraph is the stale original, but the + // shadow-relayed delta only inserts the new paragraph — so the peer's " EDITED" must survive. + applyAgentStreamFrame( + editor, + session, + 'Alpha paragraph.\n\nBeta paragraph.\n\nGamma paragraph.' + ) + endAgentStream(session) + + const live = doc.getXmlFragment('default').toString() + expect(live).toContain('EDITED') + expect(live).toContain('Gamma paragraph') }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts index a92f11110d6..dc96c0e6e58 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts @@ -1,8 +1,12 @@ import type { Editor } from '@tiptap/core' import { Node as PMNode } from '@tiptap/pm/model' -import { updateYFragment, ySyncPluginKey } from '@tiptap/y-tiptap' +import { initProseMirrorDoc, updateYFragment, ySyncPluginKey } from '@tiptap/y-tiptap' +import * as Y from 'yjs' import { parseMarkdownToDoc } from '../markdown-parse' +/** The Yjs fragment name TipTap's Collaboration extension binds to (its default `field`). */ +const COLLAB_DOC_FIELD = 'default' + /** * Transaction origin for agent-streamed writes into a live collaborative doc. It is deliberately NOT * the `ySyncPluginKey` origin that local user edits use, so the Collaboration UndoManager — which @@ -11,22 +15,64 @@ import { parseMarkdownToDoc } from '../markdown-parse' const AGENT_STREAM_ORIGIN = Symbol('agent-stream') /** - * Apply a streamed markdown body into the editor's live collaborative Y.Doc as a minimal CRDT diff. - * - * Uses the running `ySyncPlugin` binding's {@link updateYFragment} — the same primitive TipTap runs on - * every keystroke — so only the delta between the doc's current content and `body` is written, never a - * full-document replace that would wipe collaborators. Each diff is a small Yjs op that renders locally - * (via the binding's observer, the remote-edit render path) AND broadcasts to every peer, so the stream - * is smooth here and on other clients alike. Runs under {@link AGENT_STREAM_ORIGIN} so the streamed ops - * stay out of the user's undo stack. Returns `false` when the editor has no live ySync binding (e.g. a - * non-collaborative editor); the caller gates seed-readiness separately via `collabReady`. + * A private Yjs replica the agent stream reconciles against, so a stream writes into the live doc as a + * TRUE peer: only the agent's own delta reaches the shared doc, never a whole-document reconcile that + * would revert a collaborator's concurrent edit. Seeded from the live doc at stream start; it receives + * ONLY agent reconciles (never peer updates), so `shadow → nextTarget` yields exactly the agent's change. + */ +export interface AgentStreamSession { + shadow: Y.Doc + fragment: Y.XmlFragment +} + +/** + * Begin an agent stream by snapshotting the live doc into a private shadow replica. Returns `null` when + * the editor has no live ySync binding (e.g. a non-collaborative editor). + */ +export function beginAgentStream(editor: Editor): AgentStreamSession | null { + const binding = ySyncPluginKey.getState(editor.state)?.binding + if (!binding) return null + const shadow = new Y.Doc() + Y.applyUpdate(shadow, Y.encodeStateAsUpdate(binding.doc)) + return { shadow, fragment: shadow.getXmlFragment(COLLAB_DOC_FIELD) } +} + +/** + * Apply one streamed markdown body. Reconciles the shadow toward `body` with `updateYFragment` (the same + * minimal-diff primitive TipTap runs per keystroke), captures ONLY the resulting agent delta, and relays + * it into the live doc under {@link AGENT_STREAM_ORIGIN}. Because the shadow never sees peer updates, the + * delta touches only what the agent changed — so concurrent peer edits elsewhere in the live doc survive, + * the change renders locally (via the binding's observer, the remote-edit path), broadcasts to every + * peer, and stays out of the user's undo stack. Returns `false` when the editor has no live ySync binding. */ -export function applyStreamedMarkdownToLiveDoc(editor: Editor, body: string): boolean { +export function applyAgentStreamFrame( + editor: Editor, + session: AgentStreamSession, + body: string +): boolean { const binding = ySyncPluginKey.getState(editor.state)?.binding if (!binding) return false const target = PMNode.fromJSON(editor.schema, parseMarkdownToDoc(body)) - binding.doc.transact(() => { - updateYFragment(binding.doc, binding.type, target, binding) - }, AGENT_STREAM_ORIGIN) + let delta: Uint8Array | null = null + const capture = (update: Uint8Array, origin: unknown) => { + if (origin === AGENT_STREAM_ORIGIN) delta = update + } + session.shadow.on('update', capture) + try { + session.shadow.transact(() => { + // `updateYFragment` diffs against the fragment's CURRENT content, so it needs the fragment↔PM + // binding metadata; `initProseMirrorDoc` reconstructs it from the fragment's present state. + const { meta } = initProseMirrorDoc(session.fragment, editor.schema) + updateYFragment(session.shadow, session.fragment, target, meta) + }, AGENT_STREAM_ORIGIN) + } finally { + session.shadow.off('update', capture) + } + if (delta) Y.applyUpdate(binding.doc, delta, AGENT_STREAM_ORIGIN) return true } + +/** End an agent stream and free its shadow replica. */ +export function endAgentStream(session: AgentStreamSession): void { + session.shadow.destroy() +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 540fc955301..b1f8270df8d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -20,7 +20,12 @@ import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' import { PreviewLoadingFrame } from '../preview-shared' import { useEditableFileContent } from '../use-editable-file-content' -import { applyStreamedMarkdownToLiveDoc } from './collaboration/apply-streamed-markdown' +import { + type AgentStreamSession, + applyAgentStreamFrame, + beginAgentStream, + endAgentStream, +} from './collaboration/apply-streamed-markdown' import { useFileDocCollaboration } from './collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from './editor-extensions' import { findHeadingPos } from './heading-anchors' @@ -79,6 +84,13 @@ interface RichMarkdownEditorProps { * applied live; a rebuild is only revealed while it extends what's shown (see the streaming tick). */ streamIsIncremental?: boolean + /** + * The agent edit operation driving the stream, when known (`create`/`append`/`update`/`patch`). Used + * only to relax the "must extend" gate for `patch` (which legitimately replaces a mid-document region): + * every other operation's snapshot must extend what's shown, so a base-less `append` fragment can't + * reconcile the live doc to a wipe. + */ + streamOperation?: string disableStreamingAutoScroll?: boolean previewContextKey?: string /** Disable the `@` tag-insertion menu (existing tags still render). Defaults off — the file editor keeps tagging. */ @@ -86,7 +98,7 @@ interface RichMarkdownEditorProps { /** * Opt this surface into live collaborative editing (Files page + the embedded chat file preview). * Collaboration can coexist with agent streaming: while streaming, the growing content is applied to - * the shared Y.Doc as minimal CRDT diffs (see {@link applyStreamedMarkdownToLiveDoc}) rather than a + * the shared Y.Doc as minimal CRDT diffs (see {@link applyAgentStreamFrame}) rather than a * full-document `setContent`, so the stream stays smooth and every peer sees it live. */ collaborative?: boolean @@ -111,6 +123,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ streamingContent, isAgentEditing, streamIsIncremental, + streamOperation, disableStreamingAutoScroll = false, previewContextKey, disableTagging, @@ -181,6 +194,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ userName={userName} autoFocus={autoFocus} streamIsIncremental={streamIsIncremental} + streamOperation={streamOperation} disableStreamingAutoScroll={disableStreamingAutoScroll} disableTagging={disableTagging} collaborative={collaborative} @@ -206,6 +220,8 @@ interface LoadedRichMarkdownEditorProps { autoFocus?: boolean /** See {@link RichMarkdownEditorProps.streamIsIncremental}. */ streamIsIncremental?: boolean + /** See {@link RichMarkdownEditorProps.streamOperation}. */ + streamOperation?: string disableStreamingAutoScroll?: boolean disableTagging?: boolean /** See {@link RichMarkdownEditorProps.collaborative}. */ @@ -239,6 +255,7 @@ export function LoadedRichMarkdownEditor({ userName, autoFocus, streamIsIncremental, + streamOperation, disableStreamingAutoScroll, disableTagging, collaborative = false, @@ -356,6 +373,10 @@ export function LoadedRichMarkdownEditor({ */ const streamIsIncrementalRef = useRef(streamIsIncremental) streamIsIncrementalRef.current = streamIsIncremental + const streamOperationRef = useRef(streamOperation) + streamOperationRef.current = streamOperation + /** The live agent-stream shadow replica, held for the current stream and freed on settle/unmount. */ + const agentStreamSessionRef = useRef(null) const router = useRouter() const routerRef = useRef(router) routerRef.current = router @@ -855,7 +876,11 @@ export function LoadedRichMarkdownEditor({ } const shownBody = lastSyncedBodyRef.current const extendsShown = shownBody === null || pending.startsWith(shownBody) - if (!streamIsIncrementalRef.current && !extendsShown) { + // Every snapshot except a mid-document `patch` must EXTEND what's shown: a from-scratch rebuild + // (`create`/`update`) is only revealed as it grows, and an `append` snapshot that doesn't extend + // the base is a base-less fragment (the server emits one before the base loads) which would + // reconcile the seeded doc down to a wipe. Only `patch` legitimately replaces a mid-region. + if (!extendsShown && streamOperationRef.current !== 'patch') { streamRafRef.current = null return } @@ -868,9 +893,11 @@ export function LoadedRichMarkdownEditor({ } const el = containerRef.current const pinnedToBottom = el ? el.scrollHeight - el.scrollTop - el.clientHeight < 80 : false + agentStreamSessionRef.current ??= beginAgentStream(editor) + const session = agentStreamSessionRef.current // Defensive: a ready collab editor always has a ySync binding, so this applies; if one is // somehow absent, bail this frame without advancing rather than looping. - if (!applyStreamedMarkdownToLiveDoc(editor, pending)) { + if (!session || !applyAgentStreamFrame(editor, session, pending)) { streamRafRef.current = null return } @@ -892,13 +919,15 @@ export function LoadedRichMarkdownEditor({ if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false const finalBody = splitFrontmatter(content).body - if (finalBody !== lastSyncedBodyRef.current) { - runOffRender(() => { - if (applyStreamedMarkdownToLiveDoc(editor, finalBody)) { + const session = agentStreamSessionRef.current + agentStreamSessionRef.current = null + runOffRender(() => { + if (session && finalBody !== lastSyncedBodyRef.current) { + if (applyAgentStreamFrame(editor, session, finalBody)) lastSyncedBodyRef.current = finalBody - } - }) - } + } + if (session) endAgentStream(session) + }) } return } @@ -1002,6 +1031,10 @@ export function LoadedRichMarkdownEditor({ useEffect( () => () => { if (streamRafRef.current !== null) cancelAnimationFrame(streamRafRef.current) + if (agentStreamSessionRef.current) { + endAgentStream(agentStreamSessionRef.current) + agentStreamSessionRef.current = null + } }, [] ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index 7af698dd71d..2b063f54dc4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -250,6 +250,7 @@ export const ResourceContent = memo(function ResourceContent({ } isAgentEditing={isAgentEditing} streamIsIncremental={streamIsIncremental} + streamOperation={previewSession?.operation} disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} /> @@ -663,6 +664,7 @@ interface EmbeddedFileProps { streamingContent?: string isAgentEditing?: boolean streamIsIncremental?: boolean + streamOperation?: string disableStreamingAutoScroll?: boolean previewContextKey?: string } @@ -675,6 +677,7 @@ function EmbeddedFile({ streamingContent, isAgentEditing, streamIsIncremental, + streamOperation, disableStreamingAutoScroll = false, previewContextKey, }: EmbeddedFileProps) { @@ -718,6 +721,7 @@ function EmbeddedFile({ streamingContent={streamingContent} isAgentEditing={isAgentEditing} streamIsIncremental={streamIsIncremental} + streamOperation={streamOperation} disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} collaborative From 7b2bc180dbeb31d43ca4f7ed7277ffc894f0dfae Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 30 Jul 2026 23:21:37 -0700 Subject: [PATCH 03/15] fix(files): destroy the agent shadow deterministically on settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 1 (Low): endAgentStream ran inside runOffRender, whose microtask is dropped when a rapid follow-up stream bumps the run token — leaking the shadow Y.Doc. Split it out into an unguarded microtask queued after the (droppable) final apply, so the shadow is always destroyed. --- .../rich-markdown-editor.tsx | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index b1f8270df8d..80eb18f825e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -921,13 +921,20 @@ export function LoadedRichMarkdownEditor({ const finalBody = splitFrontmatter(content).body const session = agentStreamSessionRef.current agentStreamSessionRef.current = null - runOffRender(() => { - if (session && finalBody !== lastSyncedBodyRef.current) { - if (applyAgentStreamFrame(editor, session, finalBody)) + if (session) { + runOffRender(() => { + if ( + finalBody !== lastSyncedBodyRef.current && + applyAgentStreamFrame(editor, session, finalBody) + ) { lastSyncedBodyRef.current = finalBody - } - if (session) endAgentStream(session) - }) + } + }) + // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream + // can supersede the run token and drop the apply above, but the shadow must always be + // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. + queueMicrotask(() => endAgentStream(session)) + } } return } From 6b23e8acf4b6786292db2915dbbd4811c1e6e4c4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 30 Jul 2026 23:54:54 -0700 Subject: [PATCH 04/15] fix(files): agent stream frames skip the relay's durable persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 1 (High): client-applied stream frames broadcast over the sync channel, so the relay stamped a socket origin and ran schedulePersist — durably writing partial agent content mid-stream, attributed to the watching user (the old server-merge applied with no origin and never did). Restore that behavior: - new FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST wire tag; the provider tags AGENT_STREAM_ORIGIN updates with it (normal user edits stay SYNC) - the relay applies it under an AgentSyncOrigin (carries the socket id for broadcast exclusion, but is not a plain string) so originSocketId() is null → no edited/schedulePersist/lastEditorUserId; excludeSocketId() still excludes the sender, and the update still publishes to the stream so peers converge - the copilot's final edit_content write remains the authoritative durable persist - tests: relay applies+fans-out but never persists a SYNC_NO_PERSIST frame (verified it fails if applied as a socket edit); provider tags agent edits --- apps/realtime/src/handlers/file-doc.test.ts | 30 +++++++++++++ apps/realtime/src/handlers/file-doc.ts | 42 +++++++++++++++++-- .../collaboration/apply-streamed-markdown.ts | 2 +- .../collaboration/file-doc-provider.test.ts | 14 +++++++ .../collaboration/file-doc-provider.ts | 10 ++++- packages/realtime-protocol/src/file-doc.ts | 9 ++++ 6 files changed, 102 insertions(+), 5 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index a6f59df67bf..7ca2ab42831 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -299,6 +299,36 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocPersist).toHaveBeenCalled() }) + it('applies + fans out an agent-streamed frame (SYNC_NO_PERSIST) but never persists it', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) + const { io, sent } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + const before = sent.length + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'agent streamed this') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(edit)) + ) + ) + await flushMicrotasks() + + // It fans out to the room excluding the sender, so a collaborator sees the stream live... + const fanout = sent + .slice(before) + .filter((m) => m.event === FILE_DOC_EVENTS.MESSAGE && m.except === 'socket-1') + expect(fanout.length).toBeGreaterThan(0) + + // ...but it must NOT mark the doc dirty: a last-disconnect flush never persists agent content (the + // copilot's final edit_content write is the authoritative durable persist). + cleanupFileDocForSocket('socket-1', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + }) + it('stops on a persist conflict without clobbering (single attempt, durable left authoritative)', async () => { mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) // A persist reports an out-of-band change (If-Match conflict). The relay must NOT re-persist against diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 7e502fc0582..f6f8264b9ce 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -177,6 +177,25 @@ function originSocketId(origin: unknown): string | null { return typeof origin === 'string' ? origin : null } +/** + * The transaction origin stamped on an agent-streamed frame (a {@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST} + * apply). It carries the emitting socket id for broadcast exclusion, but is deliberately NOT a plain + * string — so `originSocketId` returns `null` for it and the update never triggers `edited`/`schedulePersist` + * (the copilot's final `edit_content` write is the durable persist). + */ +interface AgentSyncOrigin { + readonly agentSocketId: string +} + +function isAgentSyncOrigin(origin: unknown): origin is AgentSyncOrigin { + return typeof origin === 'object' && origin !== null && 'agentSocketId' in origin +} + +/** The socket id to exclude when relaying an update — a client socket edit OR an agent-streamed frame. */ +function excludeSocketId(origin: unknown): string | null { + return originSocketId(origin) ?? (isAgentSyncOrigin(origin) ? origin.agentSocketId : null) +} + /** * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness * (cursors/selection) is ephemeral and needs no convergence or replay, so the adapter's cross-task @@ -684,9 +703,10 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) - // Fan out to THIS task's clients only (excluding the origin socket if local). Cross-task delivery - // rides the shared stream — every task's tailer applies + runs its own local fan-out. - broadcastLocal(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + // Fan out to THIS task's clients only (excluding the origin socket if local — a user edit OR an + // agent-streamed frame). Cross-task delivery rides the shared stream — every task's tailer applies + + // runs its own local fan-out. + broadcastLocal(io, name, encoding.toUint8Array(encoder), excludeSocketId(origin)) // Share every locally-originated update to the stream so peers converge. Skip updates that already // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN) and SEED_ORIGIN — the seed is published // EXPLICITLY and AWAITED under the seed lock (so it lands before the lock releases), which a @@ -781,6 +801,22 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { } break } + case FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST: { + // An agent-streamed frame: apply + fan out to peers (so a collaborator sees the stream live) but + // do NOT treat it as a durable user edit. Unlike SYNC, we do NOT set `lastEditorUserId`, and the + // apply uses an {@link AgentSyncOrigin} (not the bare socket id) so `originSocketId` is `null` in + // `doc.on('update')` — skipping `edited`/`schedulePersist`. `excludeSocketId` still reads the + // carried socket id, so the sender is excluded from the relay fan-out. The copilot's final + // `edit_content` write remains the authoritative durable persist. + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + const agentOrigin: AgentSyncOrigin = { agentSocketId: socket.id } + syncProtocol.readSyncMessage(decoder, encoder, room.doc, agentOrigin) + if (encoding.length(encoder) > 1) { + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + break + } case FILE_DOC_MESSAGE_TYPE.AWARENESS: { const update = decoding.readVarUint8Array(decoder) // Enforce presence ownership: a socket may only publish/remove awareness diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts index dc96c0e6e58..f0a69769b2f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.ts @@ -12,7 +12,7 @@ const COLLAB_DOC_FIELD = 'default' * the `ySyncPluginKey` origin that local user edits use, so the Collaboration UndoManager — which * tracks only `ySyncPluginKey` — excludes streamed ops from the user's undo stack. */ -const AGENT_STREAM_ORIGIN = Symbol('agent-stream') +export const AGENT_STREAM_ORIGIN = Symbol('agent-stream') /** * A private Yjs replica the agent stream reconciles against, so a stream writes into the live doc as a diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index af4346011ff..689c2ee6c77 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -12,6 +12,7 @@ import { describe, expect, it, vi } from 'vitest' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' +import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' import { FileDocProvider } from './file-doc-provider' /** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ @@ -125,6 +126,19 @@ describe('FileDocProvider', () => { expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) }) + it('tags agent-streamed edits as SYNC_NO_PERSIST so the relay skips the durable persist', () => { + const { doc, emit } = createProvider(true) + emit.mockClear() + + // An agent-streamed frame is applied under AGENT_STREAM_ORIGIN; it must still reach the server (peers + // see it live) but as SYNC_NO_PERSIST, so the relay fans it out without treating it as a user edit. + doc.transact(() => doc.getText('default').insert(0, 'agent'), AGENT_STREAM_ORIGIN) + + const messages = emittedMessages(emit) + expect(messages.length).toBe(1) + expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST) + }) + it('does not echo updates it applied from the server', () => { const { provider, emit, fire } = createProvider(true) fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index d23b3b21d49..9b55ac7b907 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -14,6 +14,7 @@ import type { Socket } from 'socket.io-client' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import type * as Y from 'yjs' +import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' /** * Events emitted by {@link FileDocProvider}. @@ -276,8 +277,15 @@ export class FileDocProvider extends ObservableV2 { if (this.fatal) return // Updates we applied from the server carry `this` as origin — don't echo them. if (origin === this) return + // Agent-streamed frames must reach peers (so a collaborator sees the stream live) but must NOT be + // treated by the server as a durable user edit — the copilot's final `edit_content` write is the + // authoritative persist. Tag them so the relay applies + fans out but skips persist bookkeeping. + const messageType = + origin === AGENT_STREAM_ORIGIN + ? FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST + : FILE_DOC_MESSAGE_TYPE.SYNC const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + encoding.writeVarUint(encoder, messageType) syncProtocol.writeUpdate(encoder, update) this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index e58ceb477a3..6746e94e97c 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -42,6 +42,15 @@ export const FILE_DOC_EVENTS = { export const FILE_DOC_MESSAGE_TYPE = { SYNC: 0, AWARENESS: 1, + /** + * Client → server: a Yjs sync UPDATE (same framing as {@link FILE_DOC_MESSAGE_TYPE.SYNC}) that the + * server must apply and fan out to peers WITHOUT treating it as a durable user edit — no + * `schedulePersist`, no `edited`/`lastEditorUserId` bookkeeping. Used for agent-streamed frames: the + * copilot's final `edit_content` write is the authoritative durable persist, so the live stream must + * not also durably write partial content (attributed to the watching user). The server never sends + * this type; replies always use {@link FILE_DOC_MESSAGE_TYPE.SYNC}. + */ + SYNC_NO_PERSIST: 2, } as const export type FileDocMessageType = (typeof FILE_DOC_MESSAGE_TYPE)[keyof typeof FILE_DOC_MESSAGE_TYPE] From db571c4059104b09014ca1a594d3a18be67d9b7d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 00:10:58 -0700 Subject: [PATCH 05/15] fix(files): open the stream shadow at start + private extend baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 2: - High (settle skips apply without session): the stream shadow is now opened on the first ready frame, BEFORE the extend gate — so an `update` rewrite (whose every frame is gated out until settle) and a stream that finishes before seed still get a session, and settle applies the final body via the reused-or-on-demand shadow instead of leaving the doc stale until the durable reconcile. - Medium (peer edits stall the stream): the extend gate now reads a private `lastStreamedBodyRef` (the agent's own last frame), snapshotted at stream start, not `lastSyncedBodyRef` which `onUpdate` clobbers on peer edits — so a collaborator typing can't make the growing snapshot stop prefixing the shown body and freeze it. - Medium (multi-replica over-persist): pre-existing, documented "safe over-persist" (a peer task tails the frame as REDIS_ORIGIN and marks edited) — refreshed the stale comment to describe the SYNC_NO_PERSIST source; copilot's edit_content write remains the authoritative durable persist. --- apps/realtime/src/handlers/file-doc.ts | 10 ++-- .../rich-markdown-editor.tsx | 55 ++++++++++++------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index f6f8264b9ce..f3b51a0026e 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -719,10 +719,12 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // snapshot on catch-up (REDIS_SNAPSHOT_ORIGIN) also counts: it folds real edits into one frame, so a // fresh task catching up purely from it must not treat the doc as unedited. The seed transition // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. NOTE: - // in the multi-replica path a copilot merge is NOT excluded — it round-trips through the stream as - // REDIS_ORIGIN, indistinguishable from a peer edit, so it marks `edited`. That only ever causes an - // extra idempotent persist of content copilot already wrote directly (safe over-persist, never a lost - // edit); the single-replica path applies the merge locally with no origin and does not count it. + // an agent-streamed frame ({@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST}) is not counted on the + // ORIGINATING task (it applies under an AgentSyncOrigin — see the handler), but in the multi-replica + // path it is published to the stream and PEER tasks apply it as REDIS_ORIGIN, indistinguishable from a + // peer edit, so it marks `edited` there. That only ever causes an extra idempotent persist of content + // the copilot's final `edit_content` write persists durably anyway (safe over-persist, never a lost + // edit); fully suppressing it would require tagging the stream entry as no-persist across replicas. const seededBefore = room.seededObserved if (isDocSeeded(room.doc)) room.seededObserved = true if ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 80eb18f825e..41cb5c94694 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -338,6 +338,14 @@ export function LoadedRichMarkdownEditor({ const lastSyncedBodyRef = useRef( streamingAtMountRef.current ? null : splitFrontmatter(content).body ) + /** + * The body the AGENT last streamed into the collaborative doc — the extend-gate baseline for the collab + * streaming path. Written ONLY at stream start (snapshotting the pre-stream base) and by the streaming + * tick, never by `onUpdate`, so a concurrent PEER edit (which does clobber {@link lastSyncedBodyRef} via + * `onUpdate`) can't make the agent's growing snapshot stop prefixing the shown body and stall the stream. + * Reset to `null` on settle so the next stream re-captures its own baseline. + */ + const lastStreamedBodyRef = useRef(null) const onChangeRef = useRef(onChange) onChangeRef.current = onChange const onSaveShortcutRef = useRef(onSaveShortcut) @@ -864,22 +872,32 @@ export function LoadedRichMarkdownEditor({ // re-runs and applies once it lands (the read-only placeholder shows the base content meanwhile — // see `showPlaceholder`). if (!collabReady) return + // Open the stream's shadow on the FIRST ready frame — BEFORE the extend gate — so its shadow + // captures the pre-stream base (immune to later peer edits) even for an `update` whose every frame + // is gated out until settle. Snapshot that base as the agent's private extend-gate baseline. + if (agentStreamSessionRef.current === null) { + agentStreamSessionRef.current = beginAgentStream(editor) + lastStreamedBodyRef.current = lastSyncedBodyRef.current + } + const session = agentStreamSessionRef.current const body = splitFrontmatter(content).body - if (body === lastSyncedBodyRef.current) return + if (body === lastStreamedBodyRef.current) return pendingStreamBodyRef.current = body if (streamRafRef.current !== null) return const tick = () => { const pending = pendingStreamBodyRef.current - if (pending === null || pending === lastSyncedBodyRef.current) { + if (pending === null || pending === lastStreamedBodyRef.current) { streamRafRef.current = null return } - const shownBody = lastSyncedBodyRef.current + const shownBody = lastStreamedBodyRef.current const extendsShown = shownBody === null || pending.startsWith(shownBody) - // Every snapshot except a mid-document `patch` must EXTEND what's shown: a from-scratch rebuild - // (`create`/`update`) is only revealed as it grows, and an `append` snapshot that doesn't extend - // the base is a base-less fragment (the server emits one before the base loads) which would - // reconcile the seeded doc down to a wipe. Only `patch` legitimately replaces a mid-region. + // Every snapshot except a mid-document `patch` must EXTEND what the AGENT last streamed: a + // from-scratch rebuild (`create`/`update`) is only revealed as it grows, and an `append` + // snapshot that doesn't extend the base is a base-less fragment (the server emits one before the + // base loads) which would reconcile the seeded doc down to a wipe. Only `patch` replaces a + // mid-region. The baseline is the agent's own last frame (not the editor body), so a concurrent + // peer edit can't make the growing snapshot stop prefixing it and stall the stream. if (!extendsShown && streamOperationRef.current !== 'patch') { streamRafRef.current = null return @@ -893,8 +911,6 @@ export function LoadedRichMarkdownEditor({ } const el = containerRef.current const pinnedToBottom = el ? el.scrollHeight - el.scrollTop - el.clientHeight < 80 : false - agentStreamSessionRef.current ??= beginAgentStream(editor) - const session = agentStreamSessionRef.current // Defensive: a ready collab editor always has a ySync binding, so this applies; if one is // somehow absent, bail this frame without advancing rather than looping. if (!session || !applyAgentStreamFrame(editor, session, pending)) { @@ -902,7 +918,7 @@ export function LoadedRichMarkdownEditor({ return } streamRafRef.current = null - lastSyncedBodyRef.current = pending + lastStreamedBodyRef.current = pending lastStreamParseAtRef.current = performance.now() if (!disableStreamingAutoScroll && el && pinnedToBottom) el.scrollTop = el.scrollHeight } @@ -913,22 +929,20 @@ export function LoadedRichMarkdownEditor({ cancelAnimationFrame(streamRafRef.current) streamRafRef.current = null } - // Settle: once seeded, apply the final body once (the last streaming frame may have been throttled) - // so the Y.Doc exactly equals the streamed result; the durable server write then lands as a noop - // diff. If the seed has not arrived, keep `wasStreamingRef` set so the post-seed re-run applies. + // Settle: apply the FINAL body once so the Y.Doc exactly equals the streamed result — even when + // every mid-stream frame was gated out (an `update` rewrite never extends the base) or the stream + // finished before the seed, cases where no frame applied. Reuse the stream's shadow when it exists + // (seeded from the pre-stream base, so peer edits survive); otherwise open one on demand. The + // durable server write then lands as a noop diff. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false const finalBody = splitFrontmatter(content).body - const session = agentStreamSessionRef.current + const session = agentStreamSessionRef.current ?? beginAgentStream(editor) agentStreamSessionRef.current = null + lastStreamedBodyRef.current = null if (session) { runOffRender(() => { - if ( - finalBody !== lastSyncedBodyRef.current && - applyAgentStreamFrame(editor, session, finalBody) - ) { - lastSyncedBodyRef.current = finalBody - } + applyAgentStreamFrame(editor, session, finalBody) }) // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream // can supersede the run token and drop the apply above, but the shadow must always be @@ -1042,6 +1056,7 @@ export function LoadedRichMarkdownEditor({ endAgentStream(agentStreamSessionRef.current) agentStreamSessionRef.current = null } + lastStreamedBodyRef.current = null }, [] ) From eb2db57bca51a916cc9c4363cb879fd29ab011f0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 00:43:04 -0700 Subject: [PATCH 06/15] fix(files): fail-close base-less previews + operation-based stream hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor/Greptile round 3 (High + Medium) — remove the fragile string-prefix "extend gate", which was the root of both findings: - Server: `buildFilePreviewText` now fails closed for an `append` whose base content hasn't loaded (returns undefined, like patch/update), so a base-less fragment never reaches the client. This eliminates the base-less wipe at settle (Greptile P1) at the source; an empty file (existingContent === '') still previews normally. - Client: the collab streaming tick no longer string-prefixes the raw preview against the editor's canonical markdown (the '*' vs '-' / emphasis mismatch that froze every append frame — Cursor). The mid-stream hold is now purely operation-based: `update` waits for settle; append/patch/create apply each frame via the (peer-safe) shadow reconcile. lastStreamedBodyRef is now a plain dedup guard, not a prefix baseline. Keeps the shadow, durable write, and SYNC_NO_PERSIST unchanged. --- .../rich-markdown-editor.tsx | 52 ++++++++----------- .../tools/server/files/file-preview.test.ts | 23 ++++++++ .../tools/server/files/file-preview.ts | 11 ++-- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 41cb5c94694..3f35b6a4d4c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -85,10 +85,10 @@ interface RichMarkdownEditorProps { */ streamIsIncremental?: boolean /** - * The agent edit operation driving the stream, when known (`create`/`append`/`update`/`patch`). Used - * only to relax the "must extend" gate for `patch` (which legitimately replaces a mid-document region): - * every other operation's snapshot must extend what's shown, so a base-less `append` fragment can't - * reconcile the live doc to a wipe. + * The agent edit operation driving the stream, when known (`create`/`append`/`update`/`patch`). In the + * collaborative path it decides only whether to stream mid-flight: an `update` (from-scratch rewrite) is + * HELD until settle so the open doc doesn't collapse to a partial result, while `append`/`patch`/`create` + * apply each frame. */ streamOperation?: string disableStreamingAutoScroll?: boolean @@ -339,11 +339,11 @@ export function LoadedRichMarkdownEditor({ streamingAtMountRef.current ? null : splitFrontmatter(content).body ) /** - * The body the AGENT last streamed into the collaborative doc — the extend-gate baseline for the collab - * streaming path. Written ONLY at stream start (snapshotting the pre-stream base) and by the streaming - * tick, never by `onUpdate`, so a concurrent PEER edit (which does clobber {@link lastSyncedBodyRef} via - * `onUpdate`) can't make the agent's growing snapshot stop prefixing the shown body and stall the stream. - * Reset to `null` on settle so the next stream re-captures its own baseline. + * The body the AGENT last applied into the collaborative doc — a dedup guard for the collab streaming + * tick, so an unchanged frame skips a redundant shadow reconcile/reparse. Written ONLY by the streaming + * tick (never by `onUpdate`), and reset to `null` on settle for the next stream. It is NOT a string-prefix + * baseline: the mid-stream hold is decided by operation (`update` waits for settle), not by comparing the + * raw preview against the editor's canonical markdown. */ const lastStreamedBodyRef = useRef(null) const onChangeRef = useRef(onChange) @@ -872,13 +872,10 @@ export function LoadedRichMarkdownEditor({ // re-runs and applies once it lands (the read-only placeholder shows the base content meanwhile — // see `showPlaceholder`). if (!collabReady) return - // Open the stream's shadow on the FIRST ready frame — BEFORE the extend gate — so its shadow - // captures the pre-stream base (immune to later peer edits) even for an `update` whose every frame - // is gated out until settle. Snapshot that base as the agent's private extend-gate baseline. - if (agentStreamSessionRef.current === null) { - agentStreamSessionRef.current = beginAgentStream(editor) - lastStreamedBodyRef.current = lastSyncedBodyRef.current - } + // Open the stream's shadow on the FIRST ready frame so it captures the pre-stream base (immune to + // later peer edits) — including for an `update`, whose frames are all held until settle, so settle + // still has a shadow through which to apply the final rewrite. + agentStreamSessionRef.current ??= beginAgentStream(editor) const session = agentStreamSessionRef.current const body = splitFrontmatter(content).body if (body === lastStreamedBodyRef.current) return @@ -890,15 +887,11 @@ export function LoadedRichMarkdownEditor({ streamRafRef.current = null return } - const shownBody = lastStreamedBodyRef.current - const extendsShown = shownBody === null || pending.startsWith(shownBody) - // Every snapshot except a mid-document `patch` must EXTEND what the AGENT last streamed: a - // from-scratch rebuild (`create`/`update`) is only revealed as it grows, and an `append` - // snapshot that doesn't extend the base is a base-less fragment (the server emits one before the - // base loads) which would reconcile the seeded doc down to a wipe. Only `patch` replaces a - // mid-region. The baseline is the agent's own last frame (not the editor body), so a concurrent - // peer edit can't make the growing snapshot stop prefixing it and stall the stream. - if (!extendsShown && streamOperationRef.current !== 'patch') { + // Hold a from-scratch rewrite (`update`) until settle so the open doc doesn't collapse to a + // partial rewrite mid-stream (matching `main`). `append`/`patch`/`create` apply each frame — the + // shadow reconcile is peer-safe, and base-less `append` fragments no longer reach the client (the + // server fail-closes them), so there is nothing here to string-prefix or wipe-guard against. + if (streamOperationRef.current === 'update') { streamRafRef.current = null return } @@ -929,11 +922,10 @@ export function LoadedRichMarkdownEditor({ cancelAnimationFrame(streamRafRef.current) streamRafRef.current = null } - // Settle: apply the FINAL body once so the Y.Doc exactly equals the streamed result — even when - // every mid-stream frame was gated out (an `update` rewrite never extends the base) or the stream - // finished before the seed, cases where no frame applied. Reuse the stream's shadow when it exists - // (seeded from the pre-stream base, so peer edits survive); otherwise open one on demand. The - // durable server write then lands as a noop diff. + // Settle: apply the FINAL body once so the Y.Doc exactly equals the streamed result — even when no + // frame applied mid-stream (an `update` is held until settle, or the stream finished before the + // seed). Reuse the stream's shadow when it exists (seeded from the pre-stream base, so peer edits + // survive); otherwise open one on demand. The durable server write then lands as a noop diff. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false const finalBody = splitFrontmatter(content).body diff --git a/apps/sim/lib/copilot/tools/server/files/file-preview.test.ts b/apps/sim/lib/copilot/tools/server/files/file-preview.test.ts index 52a033c1b62..9e677b51377 100644 --- a/apps/sim/lib/copilot/tools/server/files/file-preview.test.ts +++ b/apps/sim/lib/copilot/tools/server/files/file-preview.test.ts @@ -31,6 +31,29 @@ describe('buildFilePreviewText', () => { ).toBe('line one\nline two') }) + it('fails closed (returns undefined) for an append when the base content has not loaded', () => { + // A base-less append preview is just the streamed fragment; a collaborative editor applying it as the + // full body would reconcile the seeded doc down to that fragment (a wipe). It must fail closed until + // the base loads, exactly like patch/update. + expect( + buildFilePreviewText({ + operation: 'append', + existingContent: undefined, + streamedContent: 'orphan fragment', + }) + ).toBeUndefined() + }) + + it('still previews an append into an EMPTY file (existingContent is "", not undefined)', () => { + expect( + buildFilePreviewText({ + operation: 'append', + existingContent: '', + streamedContent: 'first line', + }) + ).toBe('first line') + }) + it('applies anchored replace_between previews', () => { expect( buildFilePreviewText({ diff --git a/apps/sim/lib/copilot/tools/server/files/file-preview.ts b/apps/sim/lib/copilot/tools/server/files/file-preview.ts index 4ea1919395e..11b09b61c3a 100644 --- a/apps/sim/lib/copilot/tools/server/files/file-preview.ts +++ b/apps/sim/lib/copilot/tools/server/files/file-preview.ts @@ -182,10 +182,15 @@ export function buildFilePreviewText({ } if (operation === 'append') { - if (existingContent !== undefined) { - return buildAppendPreview(existingContent, streamedContent) + // Fail closed (like `patch`/`update` below) when the base file content has not loaded yet: a base-less + // `append` preview is just the streamed fragment, and a collaborative editor applying it as the full + // body would reconcile the seeded doc down to that fragment (a wipe). Skipping the preview until the + // base is available costs only a brief render delay; the final durable `edit_content` write is + // authoritative. An empty file has `existingContent === ''` (defined), so it is unaffected. + if (existingContent === undefined) { + return undefined } - return streamedContent + return buildAppendPreview(existingContent, streamedContent) } if (existingContent === undefined) { From 59a10a7b93695ecb324f98289e286f82f4958eb8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 10:10:19 -0700 Subject: [PATCH 07/15] fix(files): elect a single agent-stream writer across tabs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 4 (High): with the stream applied client-side, two tabs/windows on the same chat could each derive streamingContent (the reconnect/resume path re-consumes preview events) and each independently insert the stream under a different Yjs clientID, duplicating content until the durable reconcile. Fix — single-writer election via the file-doc awareness (new agent-stream-leader): - a client applying an agent stream announces `agentApplying` on its own awareness - only the leader (min clientID among announcers) applies mid-stream AND at settle; a non-leader renders the leader's ops via Yjs and does not apply (a non-leader applying the final body would re-insert the whole doc as a duplicate) - re-checked each frame, so it converges to one writer the moment awareness propagates; the sub-frame startup race is reconciled by the durable write - single-client (the common case) is unaffected: it is the only announcer, so it always leads --- .../collaboration/agent-stream-leader.test.ts | 55 +++++++++++++++++++ .../collaboration/agent-stream-leader.ts | 37 +++++++++++++ .../rich-markdown-editor.tsx | 41 +++++++++++--- 3 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.test.ts new file mode 100644 index 00000000000..59ae7f50649 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { + announceAgentApplying, + clearAgentApplying, + isAgentStreamLeader, +} from './agent-stream-leader' + +/** An Awareness with explicit peer states injected (self, if present, carries no `agentApplying`). */ +function awarenessWith(entries: Array<[number, Record]>): Awareness { + const aw = new Awareness(new Y.Doc()) + const states = aw.getStates() as Map> + for (const [clientId, state] of entries) states.set(clientId, state) + return aw +} + +describe('agent-stream leader election', () => { + it('a sole announcer is the leader', () => { + expect(isAgentStreamLeader(awarenessWith([[5, { agentApplying: true }]]), 5)).toBe(true) + }) + + it('the lowest clientID among announcers leads; higher announcers do not', () => { + const aw = awarenessWith([ + [7, { agentApplying: true }], + [3, { agentApplying: true }], + [9, { user: { name: 'someone else, not applying' } }], + ]) + expect(isAgentStreamLeader(aw, 3)).toBe(true) + expect(isAgentStreamLeader(aw, 7)).toBe(false) + }) + + it('a client that is not announcing is never the leader', () => { + expect(isAgentStreamLeader(awarenessWith([[3, { agentApplying: true }]]), 8)).toBe(false) + }) + + it('with no announcers, nobody leads', () => { + expect(isAgentStreamLeader(awarenessWith([[3, { user: {} }]]), 3)).toBe(false) + }) + + it('announce makes self the leader; clear relinquishes it', () => { + const doc = new Y.Doc() + const aw = new Awareness(doc) + announceAgentApplying(aw) + expect(aw.getLocalState()?.agentApplying).toBe(true) + expect(isAgentStreamLeader(aw, doc.clientID)).toBe(true) + + clearAgentApplying(aw) + expect(aw.getLocalState()?.agentApplying ?? null).toBeNull() + expect(isAgentStreamLeader(aw, doc.clientID)).toBe(false) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.ts new file mode 100644 index 00000000000..8cc79f790ac --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/agent-stream-leader.ts @@ -0,0 +1,37 @@ +import type { Awareness } from 'y-protocols/awareness' + +/** + * Awareness field a collaborative client sets on its OWN state while it is applying an agent stream into + * the shared doc. Read by every peer to run the single-writer election below. Coexists with the caret + * `user` field (`setLocalStateField` writes one field without clobbering others). + */ +const AGENT_APPLYING_FIELD = 'agentApplying' + +/** Announce that this client is applying an agent stream (candidate in the leader election). */ +export function announceAgentApplying(awareness: Awareness): void { + awareness.setLocalStateField(AGENT_APPLYING_FIELD, true) +} + +/** Stop announcing (this client is no longer applying an agent stream). */ +export function clearAgentApplying(awareness: Awareness): void { + awareness.setLocalStateField(AGENT_APPLYING_FIELD, null) +} + +/** + * Single-writer election: exactly one collaborative client applies a given agent stream into the shared + * doc, so N tabs/windows watching the same live copilot stream don't each insert it under a different + * Yjs clientID and duplicate the content. The leader is the MINIMUM clientID among all clients currently + * announcing (via {@link announceAgentApplying}) that they are applying — a deterministic tie-break that + * needs no coordinator. A brief startup race (before an announcement propagates to peers) is bounded to a + * frame or two — self-corrected the moment awareness converges, and reconciled anyway by the durable + * server write. In the common single-client case the caller is the only announcer, so it always leads. + */ +export function isAgentStreamLeader(awareness: Awareness, selfClientId: number): boolean { + let leader = Number.POSITIVE_INFINITY + awareness.getStates().forEach((state, clientId) => { + if ((state as Record | undefined)?.[AGENT_APPLYING_FIELD] === true) { + leader = Math.min(leader, clientId) + } + }) + return leader === selfClientId +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 3f35b6a4d4c..2c23b6f6e94 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -20,6 +20,11 @@ import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' import { PreviewLoadingFrame } from '../preview-shared' import { useEditableFileContent } from '../use-editable-file-content' +import { + announceAgentApplying, + clearAgentApplying, + isAgentStreamLeader, +} from './collaboration/agent-stream-leader' import { type AgentStreamSession, applyAgentStreamFrame, @@ -874,8 +879,12 @@ export function LoadedRichMarkdownEditor({ if (!collabReady) return // Open the stream's shadow on the FIRST ready frame so it captures the pre-stream base (immune to // later peer edits) — including for an `update`, whose frames are all held until settle, so settle - // still has a shadow through which to apply the final rewrite. - agentStreamSessionRef.current ??= beginAgentStream(editor) + // still has a shadow through which to apply the final rewrite. Announce candidacy in the + // single-writer election so only one tab/window actually applies this stream (see the tick). + if (agentStreamSessionRef.current === null) { + agentStreamSessionRef.current = beginAgentStream(editor) + if (collaboration) announceAgentApplying(collaboration.awareness) + } const session = agentStreamSessionRef.current const body = splitFrontmatter(content).body if (body === lastStreamedBodyRef.current) return @@ -895,6 +904,17 @@ export function LoadedRichMarkdownEditor({ streamRafRef.current = null return } + // Single-writer election: only the leader (min clientID among clients applying this stream) + // writes it into the shared doc, so multiple tabs/windows watching the same live copilot stream + // don't each insert it and duplicate content. A non-leader renders the leader's ops via Yjs; + // re-checked each frame, so it converges to one writer the moment awareness propagates. + if ( + collaboration && + !isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) + ) { + streamRafRef.current = null + return + } if ( pending.length > STREAM_REPARSE_THROTTLE_THRESHOLD && performance.now() - lastStreamParseAtRef.current < STREAM_REPARSE_THROTTLE_MS @@ -928,14 +948,21 @@ export function LoadedRichMarkdownEditor({ // survive); otherwise open one on demand. The durable server write then lands as a noop diff. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false + // Only the elected leader applies the final body — a non-leader never applied mid-stream, so + // reconciling its base-seeded shadow to the final body would re-insert the whole doc as a + // duplicate; it converges to the final state via Yjs + the durable server write instead. Compute + // leadership BEFORE clearing our announcement, then stop announcing. + const wasLeader = + !collaboration || isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) + if (collaboration) clearAgentApplying(collaboration.awareness) + lastStreamedBodyRef.current = null const finalBody = splitFrontmatter(content).body - const session = agentStreamSessionRef.current ?? beginAgentStream(editor) + const session = wasLeader + ? (agentStreamSessionRef.current ?? beginAgentStream(editor)) + : agentStreamSessionRef.current agentStreamSessionRef.current = null - lastStreamedBodyRef.current = null if (session) { - runOffRender(() => { - applyAgentStreamFrame(editor, session, finalBody) - }) + if (wasLeader) runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream // can supersede the run token and drop the apply above, but the shadow must always be // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. From 019794055f95c4ce971d8856c18b839d9545f7c3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 10:19:24 -0700 Subject: [PATCH 08/15] fix(files): gate the settle apply locally, not on a settle-time re-election MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 5 (High): the settle recomputed leadership from live awareness and the leader cleared its announcement immediately, so a straggler peer that settled afterward became the sole announcer, self-elected, and applied finalBody through its base-seeded shadow — re-inserting the whole doc as a duplicate. Fix: gate the settle apply on a LOCAL didApplyStreamRef (set only when this client actually applied a mid-stream frame — i.e. it was the mid-stream leader whose shadow is up to date), not on a settle-time re-election. A client that never applied (non-leader, a held `update`, or a pre-seed stream) skips the final apply and converges via Yjs + the durable write. The mid-stream leader election (isAgentStreamLeader) is unchanged, so exactly one client's didApplyStreamRef is ever true. --- .../rich-markdown-editor.tsx | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 2c23b6f6e94..67f0424175e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -390,6 +390,15 @@ export function LoadedRichMarkdownEditor({ streamOperationRef.current = streamOperation /** The live agent-stream shadow replica, held for the current stream and freed on settle/unmount. */ const agentStreamSessionRef = useRef(null) + /** + * True once THIS client has applied at least one mid-stream frame for the current stream — i.e. it was + * the elected leader whose shadow is up to date. Gates the settle apply LOCALLY (not on a settle-time + * re-election, which is racy: a straggler that settles after the leader clears its announcement would + * self-elect and re-insert the whole doc via its base-seeded shadow). A client that never applied + * (non-leader, a held `update`, or a pre-seed stream) converges to the final state via Yjs + the + * durable write instead. Reset on settle. + */ + const didApplyStreamRef = useRef(false) const router = useRouter() const routerRef = useRef(router) routerRef.current = router @@ -883,6 +892,7 @@ export function LoadedRichMarkdownEditor({ // single-writer election so only one tab/window actually applies this stream (see the tick). if (agentStreamSessionRef.current === null) { agentStreamSessionRef.current = beginAgentStream(editor) + didApplyStreamRef.current = false if (collaboration) announceAgentApplying(collaboration.awareness) } const session = agentStreamSessionRef.current @@ -930,6 +940,7 @@ export function LoadedRichMarkdownEditor({ streamRafRef.current = null return } + didApplyStreamRef.current = true streamRafRef.current = null lastStreamedBodyRef.current = pending lastStreamParseAtRef.current = performance.now() @@ -948,21 +959,22 @@ export function LoadedRichMarkdownEditor({ // survive); otherwise open one on demand. The durable server write then lands as a noop diff. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false - // Only the elected leader applies the final body — a non-leader never applied mid-stream, so - // reconciling its base-seeded shadow to the final body would re-insert the whole doc as a - // duplicate; it converges to the final state via Yjs + the durable server write instead. Compute - // leadership BEFORE clearing our announcement, then stop announcing. - const wasLeader = - !collaboration || isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) + // Only a client that actually applied mid-stream (the elected leader, `didApplyStreamRef`) applies + // the final body — its shadow is up to date, so this just catches a throttled last frame. A client + // that never applied has a base-seeded shadow; reconciling it to the final body would re-insert the + // whole doc as a duplicate, so it skips and converges via Yjs + the durable write. This is a LOCAL + // decision (no settle-time re-election), so a straggler can't self-elect after the leader clears. + const didApply = didApplyStreamRef.current + didApplyStreamRef.current = false if (collaboration) clearAgentApplying(collaboration.awareness) lastStreamedBodyRef.current = null - const finalBody = splitFrontmatter(content).body - const session = wasLeader - ? (agentStreamSessionRef.current ?? beginAgentStream(editor)) - : agentStreamSessionRef.current + const session = agentStreamSessionRef.current agentStreamSessionRef.current = null if (session) { - if (wasLeader) runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) + if (didApply) { + const finalBody = splitFrontmatter(content).body + runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) + } // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream // can supersede the run token and drop the apply above, but the shadow must always be // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. From 3ef00a1bef67a33303904ae895a90527ebb7610d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 10:48:48 -0700 Subject: [PATCH 09/15] fix(files): open the agent-stream shadow lazily on lead (no stale handoff) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile round 6 (P1): the leader race — (a) a mid-stream leadership handoff could apply from a stale pre-stream shadow, and (b) two tabs starting the same stream before awareness converges could both lead briefly. - (a) fixed: the shadow is now opened LAZILY in the tick, only when this client actually leads, seeded from the CURRENT doc — so a handoff successor diffs against the prior leader's ops (never a stale base) and a non-leader builds no shadow at all. Announce candidacy via a dedicated ref (decoupled from the shadow); settle still gates the final apply on didApplyStreamRef (leader-only). - (b) the pure startup race is inherent to eventually-consistent election. It is now the only residual: bounded to two tabs starting the SAME stream within the awareness-propagation window, transient (converges in a frame or two), and never persisted (SYNC_NO_PERSIST + the durable edit_content reconcile). Resumes are sequential, so the common multi-tab case elects cleanly. Documented inline; a server-granted lease would close it fully but at a round-trip cost on the common single-tab path, which isn't worth it. --- .../rich-markdown-editor.tsx | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 67f0424175e..f70beaaf98d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -399,6 +399,8 @@ export function LoadedRichMarkdownEditor({ * durable write instead. Reset on settle. */ const didApplyStreamRef = useRef(false) + /** True once this client has announced candidacy in the agent-stream election for the current stream. */ + const agentAnnouncedRef = useRef(false) const router = useRouter() const routerRef = useRef(router) routerRef.current = router @@ -886,16 +888,15 @@ export function LoadedRichMarkdownEditor({ // re-runs and applies once it lands (the read-only placeholder shows the base content meanwhile — // see `showPlaceholder`). if (!collabReady) return - // Open the stream's shadow on the FIRST ready frame so it captures the pre-stream base (immune to - // later peer edits) — including for an `update`, whose frames are all held until settle, so settle - // still has a shadow through which to apply the final rewrite. Announce candidacy in the - // single-writer election so only one tab/window actually applies this stream (see the tick). - if (agentStreamSessionRef.current === null) { - agentStreamSessionRef.current = beginAgentStream(editor) + // Announce candidacy in the single-writer election (see the tick) so only one tab/window applies + // this stream. The shadow is opened lazily in the tick, only when THIS client actually leads — so a + // non-leader builds none, and a client that takes leadership mid-stream (a handoff) seeds its shadow + // from the CURRENT doc, already carrying the prior leader's ops, never a stale base. + if (!agentAnnouncedRef.current) { + agentAnnouncedRef.current = true didApplyStreamRef.current = false if (collaboration) announceAgentApplying(collaboration.awareness) } - const session = agentStreamSessionRef.current const body = splitFrontmatter(content).body if (body === lastStreamedBodyRef.current) return pendingStreamBodyRef.current = body @@ -914,10 +915,15 @@ export function LoadedRichMarkdownEditor({ streamRafRef.current = null return } - // Single-writer election: only the leader (min clientID among clients applying this stream) - // writes it into the shared doc, so multiple tabs/windows watching the same live copilot stream - // don't each insert it and duplicate content. A non-leader renders the leader's ops via Yjs; - // re-checked each frame, so it converges to one writer the moment awareness propagates. + // Single-writer election: only the leader (min clientID among clients announcing they apply this + // stream) writes it into the shared doc, so multiple tabs/windows watching the same live copilot + // stream don't each insert it and duplicate content. A non-leader renders the leader's ops via + // Yjs; re-checked each frame, so it converges to one writer the moment awareness propagates. + // Bounded residual (accepted): if two tabs start the SAME stream within the awareness-propagation + // window they briefly both lead and duplicate a frame or two — a rare, transient, never-persisted + // glitch (SYNC_NO_PERSIST keeps it out of storage; the durable edit_content write reconciles the + // final doc). Resumes are sequential (the second tab sees the first's announcement), so the common + // multi-tab case elects cleanly. if ( collaboration && !isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) @@ -934,8 +940,11 @@ export function LoadedRichMarkdownEditor({ } const el = containerRef.current const pinnedToBottom = el ? el.scrollHeight - el.scrollTop - el.clientHeight < 80 : false - // Defensive: a ready collab editor always has a ySync binding, so this applies; if one is - // somehow absent, bail this frame without advancing rather than looping. + // Open the shadow lazily HERE — only when THIS client actually leads — seeded from the CURRENT + // doc, so a handoff successor diffs against the prior leader's ops (no stale base) and a + // non-leader never builds one. Defensive: a ready collab editor always has a ySync binding. + agentStreamSessionRef.current ??= beginAgentStream(editor) + const session = agentStreamSessionRef.current if (!session || !applyAgentStreamFrame(editor, session, pending)) { streamRafRef.current = null return @@ -953,19 +962,17 @@ export function LoadedRichMarkdownEditor({ cancelAnimationFrame(streamRafRef.current) streamRafRef.current = null } - // Settle: apply the FINAL body once so the Y.Doc exactly equals the streamed result — even when no - // frame applied mid-stream (an `update` is held until settle, or the stream finished before the - // seed). Reuse the stream's shadow when it exists (seeded from the pre-stream base, so peer edits - // survive); otherwise open one on demand. The durable server write then lands as a noop diff. + // Settle: only a client that actually applied mid-stream (the elected leader, `didApplyStreamRef`) + // applies the FINAL body — its shadow is up to date, so this just catches a throttled last frame, so + // the Y.Doc exactly equals the streamed result. A client that never applied (a non-leader, a held + // `update`, or a pre-seed stream) has no shadow and skips — it converges via Yjs + the durable write. + // This is a LOCAL decision (no settle-time re-election), so a straggler can't self-elect after the + // leader clears its announcement. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false - // Only a client that actually applied mid-stream (the elected leader, `didApplyStreamRef`) applies - // the final body — its shadow is up to date, so this just catches a throttled last frame. A client - // that never applied has a base-seeded shadow; reconciling it to the final body would re-insert the - // whole doc as a duplicate, so it skips and converges via Yjs + the durable write. This is a LOCAL - // decision (no settle-time re-election), so a straggler can't self-elect after the leader clears. const didApply = didApplyStreamRef.current didApplyStreamRef.current = false + agentAnnouncedRef.current = false if (collaboration) clearAgentApplying(collaboration.awareness) lastStreamedBodyRef.current = null const session = agentStreamSessionRef.current @@ -1088,6 +1095,8 @@ export function LoadedRichMarkdownEditor({ agentStreamSessionRef.current = null } lastStreamedBodyRef.current = null + didApplyStreamRef.current = false + agentAnnouncedRef.current = false }, [] ) From a49c118c24b84f24cada5d0a515c88803b0f7ec0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 10:54:18 -0700 Subject: [PATCH 10/15] fix(files): idempotent settle apply (update lands client-side; no straggler dup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 6 (Medium): a lone client's `update` never applied client-side — held mid-stream, then skipped by the didApplyStreamRef settle gate — so the rewrite depended entirely on the durable merge (stale if delayed/failed). Root cause was over-correcting round 5. Now that the shadow is opened lazily in the tick (current-seeded), the round-5 base-shadow duplication is already gone, so didApplyStreamRef is unnecessary. Replaced it: settle applies the final body via `agentStreamSessionRef.current ?? beginAgentStream(editor)` — the leader reuses its up-to-date shadow (last throttled frame), while a client that never applied (non-leader, held `update`, pre-seed) opens a FRESH current-seeded shadow. Reconciling current->final is idempotent: a straggler that settles after another wrote the final reconciles to a noop. So a lone `update` applies at settle (no wait on the merge), and there's still no settle-time election or base-shadow dup. --- .../rich-markdown-editor.tsx | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index f70beaaf98d..d9f5d77d845 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -390,15 +390,6 @@ export function LoadedRichMarkdownEditor({ streamOperationRef.current = streamOperation /** The live agent-stream shadow replica, held for the current stream and freed on settle/unmount. */ const agentStreamSessionRef = useRef(null) - /** - * True once THIS client has applied at least one mid-stream frame for the current stream — i.e. it was - * the elected leader whose shadow is up to date. Gates the settle apply LOCALLY (not on a settle-time - * re-election, which is racy: a straggler that settles after the leader clears its announcement would - * self-elect and re-insert the whole doc via its base-seeded shadow). A client that never applied - * (non-leader, a held `update`, or a pre-seed stream) converges to the final state via Yjs + the - * durable write instead. Reset on settle. - */ - const didApplyStreamRef = useRef(false) /** True once this client has announced candidacy in the agent-stream election for the current stream. */ const agentAnnouncedRef = useRef(false) const router = useRouter() @@ -894,7 +885,6 @@ export function LoadedRichMarkdownEditor({ // from the CURRENT doc, already carrying the prior leader's ops, never a stale base. if (!agentAnnouncedRef.current) { agentAnnouncedRef.current = true - didApplyStreamRef.current = false if (collaboration) announceAgentApplying(collaboration.awareness) } const body = splitFrontmatter(content).body @@ -949,7 +939,6 @@ export function LoadedRichMarkdownEditor({ streamRafRef.current = null return } - didApplyStreamRef.current = true streamRafRef.current = null lastStreamedBodyRef.current = pending lastStreamParseAtRef.current = performance.now() @@ -962,26 +951,23 @@ export function LoadedRichMarkdownEditor({ cancelAnimationFrame(streamRafRef.current) streamRafRef.current = null } - // Settle: only a client that actually applied mid-stream (the elected leader, `didApplyStreamRef`) - // applies the FINAL body — its shadow is up to date, so this just catches a throttled last frame, so - // the Y.Doc exactly equals the streamed result. A client that never applied (a non-leader, a held - // `update`, or a pre-seed stream) has no shadow and skips — it converges via Yjs + the durable write. - // This is a LOCAL decision (no settle-time re-election), so a straggler can't self-elect after the - // leader clears its announcement. + // Settle: apply the FINAL body so the Y.Doc exactly equals the streamed result. The mid-stream + // leader REUSES its up-to-date shadow (just catching a throttled last frame); a client that never + // applied mid-stream (a non-leader, a held `update`, or a pre-seed stream) opens a FRESH shadow + // seeded from the CURRENT doc. Reconciling current→final is idempotent — a client that settles after + // another already wrote the final reconciles to a noop — so there is NO settle-time election and no + // base-shadow duplication, and a lone client (incl. an `update`) still applies rather than waiting on + // the durable merge. That durable `edit_content` write then lands as a noop diff too. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false - const didApply = didApplyStreamRef.current - didApplyStreamRef.current = false agentAnnouncedRef.current = false if (collaboration) clearAgentApplying(collaboration.awareness) lastStreamedBodyRef.current = null - const session = agentStreamSessionRef.current + const finalBody = splitFrontmatter(content).body + const session = agentStreamSessionRef.current ?? beginAgentStream(editor) agentStreamSessionRef.current = null if (session) { - if (didApply) { - const finalBody = splitFrontmatter(content).body - runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) - } + runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream // can supersede the run token and drop the apply above, but the shadow must always be // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. @@ -1095,7 +1081,6 @@ export function LoadedRichMarkdownEditor({ agentStreamSessionRef.current = null } lastStreamedBodyRef.current = null - didApplyStreamRef.current = false agentAnnouncedRef.current = false }, [] From f3f7b7a8f269558bd29fb1801ead87fff6d3883f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:03:35 -0700 Subject: [PATCH 11/15] fix(files): broadcast agent frames to the whole room (same-socket siblings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor round 7 (Medium): SYNC_NO_PERSIST frames applied under an origin carrying the sender socket id, and excludeSocketId dropped that whole socket from the relay fan-out. A second FileDocProvider on the same socket (chat preview + Files editor) then missed all mid-stream ops and stayed stale until the durable reconcile — a regression from the old no-origin server merge, which reached both. Fix: the agent origin is now a plain AGENT_SYNC_ORIGIN symbol, and agent frames broadcast to the WHOLE room (no socket excluded), matching the old behavior — so a same-socket sibling provider stays live; the emitting provider no-ops on its own echo (the ops are already applied locally). originSocketId still returns null for the symbol, so it keeps skipping edited/schedulePersist. Removed excludeSocketId and the socket-carrying origin object. Updated the relay test to assert the whole-room broadcast (verified it fails if the sender is excluded). --- apps/realtime/src/handlers/file-doc.test.ts | 5 ++- apps/realtime/src/handlers/file-doc.ts | 45 ++++++++++----------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 7ca2ab42831..70f5351860c 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -316,10 +316,11 @@ describe('setupWorkspaceFileDocHandlers', () => { ) await flushMicrotasks() - // It fans out to the room excluding the sender, so a collaborator sees the stream live... + // It fans out to the WHOLE room — no socket excluded — so peers AND a same-socket sibling provider see + // the stream live (the emitting provider no-ops on its own echo). const fanout = sent .slice(before) - .filter((m) => m.event === FILE_DOC_EVENTS.MESSAGE && m.except === 'socket-1') + .filter((m) => m.event === FILE_DOC_EVENTS.MESSAGE && m.except === undefined) expect(fanout.length).toBeGreaterThan(0) // ...but it must NOT mark the doc dirty: a last-disconnect flush never persists agent content (the diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index f3b51a0026e..74b03b83f0e 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -179,22 +179,14 @@ function originSocketId(origin: unknown): string | null { /** * The transaction origin stamped on an agent-streamed frame (a {@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST} - * apply). It carries the emitting socket id for broadcast exclusion, but is deliberately NOT a plain - * string — so `originSocketId` returns `null` for it and the update never triggers `edited`/`schedulePersist` - * (the copilot's final `edit_content` write is the durable persist). + * apply). A non-string sentinel, so `originSocketId` returns `null` for it and the update never triggers + * `edited`/`schedulePersist` (the copilot's final `edit_content` write is the durable persist). Unlike a + * client edit, an agent frame is broadcast to the WHOLE room (its originating socket is NOT excluded), so a + * second {@link FileDocProvider} on the same socket — e.g. the chat preview alongside the Files editor — + * also receives the mid-stream ops. The emitting provider no-ops on its own echo (the ops are already + * applied locally), so broadcasting back to the sender is harmless. */ -interface AgentSyncOrigin { - readonly agentSocketId: string -} - -function isAgentSyncOrigin(origin: unknown): origin is AgentSyncOrigin { - return typeof origin === 'object' && origin !== null && 'agentSocketId' in origin -} - -/** The socket id to exclude when relaying an update — a client socket edit OR an agent-streamed frame. */ -function excludeSocketId(origin: unknown): string | null { - return originSocketId(origin) ?? (isAgentSyncOrigin(origin) ? origin.agentSocketId : null) -} +const AGENT_SYNC_ORIGIN = Symbol('file-doc-agent-sync') /** * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness @@ -706,7 +698,15 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // Fan out to THIS task's clients only (excluding the origin socket if local — a user edit OR an // agent-streamed frame). Cross-task delivery rides the shared stream — every task's tailer applies + // runs its own local fan-out. - broadcastLocal(io, name, encoding.toUint8Array(encoder), excludeSocketId(origin)) + // A client edit excludes its own sender socket (echo suppression). An agent frame broadcasts to the + // WHOLE room — no socket excluded — so a same-socket sibling provider (chat preview + Files editor) + // stays live mid-stream; the emitting provider no-ops on its own echo. + broadcastLocal( + io, + name, + encoding.toUint8Array(encoder), + origin === AGENT_SYNC_ORIGIN ? null : originSocketId(origin) + ) // Share every locally-originated update to the stream so peers converge. Skip updates that already // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN) and SEED_ORIGIN — the seed is published // EXPLICITLY and AWAITED under the seed lock (so it lands before the lock releases), which a @@ -720,7 +720,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // fresh task catching up purely from it must not treat the doc as unedited. The seed transition // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. NOTE: // an agent-streamed frame ({@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST}) is not counted on the - // ORIGINATING task (it applies under an AgentSyncOrigin — see the handler), but in the multi-replica + // ORIGINATING task (it applies under {@link AGENT_SYNC_ORIGIN} — see the handler), but in the multi-replica // path it is published to the stream and PEER tasks apply it as REDIS_ORIGIN, indistinguishable from a // peer edit, so it marks `edited` there. That only ever causes an extra idempotent persist of content // the copilot's final `edit_content` write persists durably anyway (safe over-persist, never a lost @@ -804,16 +804,15 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { break } case FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST: { - // An agent-streamed frame: apply + fan out to peers (so a collaborator sees the stream live) but + // An agent-streamed frame: apply + fan out to the room (so a collaborator sees the stream live) but // do NOT treat it as a durable user edit. Unlike SYNC, we do NOT set `lastEditorUserId`, and the - // apply uses an {@link AgentSyncOrigin} (not the bare socket id) so `originSocketId` is `null` in - // `doc.on('update')` — skipping `edited`/`schedulePersist`. `excludeSocketId` still reads the - // carried socket id, so the sender is excluded from the relay fan-out. The copilot's final + // apply uses {@link AGENT_SYNC_ORIGIN} (a non-string sentinel) so `originSocketId` is `null` in + // `doc.on('update')` — skipping `edited`/`schedulePersist`, and broadcasting to the WHOLE room + // (including the sender socket, so a same-socket sibling provider stays live). The copilot's final // `edit_content` write remains the authoritative durable persist. const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) - const agentOrigin: AgentSyncOrigin = { agentSocketId: socket.id } - syncProtocol.readSyncMessage(decoder, encoder, room.doc, agentOrigin) + syncProtocol.readSyncMessage(decoder, encoder, room.doc, AGENT_SYNC_ORIGIN) if (encoding.length(encoder) > 1) { socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } From 888496f324fd092a96d02d6e13f3530ef9819305 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:16:08 -0700 Subject: [PATCH 12/15] fix(files): tag agent stream frames no-persist across replicas A peer task tailing an agent-streamed preview frame previously applied it as REDIS_ORIGIN, marking the seeded room edited and making a transient startup-race duplicate eligible for that task's last-disconnect flush. Mark agent frames with a stream field so peers apply them as REDIS_AGENT_ORIGIN, excluded from the edited/persist gate. The copilot's durable edit_content write stays the sole authority over file bytes. --- .../src/handlers/file-doc-store.test.ts | 27 +++++++++++++- apps/realtime/src/handlers/file-doc-store.ts | 35 ++++++++++++++---- apps/realtime/src/handlers/file-doc.ts | 37 ++++++++++++------- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 7a6aba13015..9f0bf428eca 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -100,7 +100,7 @@ function makeClient(): any { vi.mock('redis', () => ({ createClient: () => makeClient() })) -import { FileDocStore } from '@/handlers/file-doc-store' +import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file-doc-store' const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' @@ -239,6 +239,31 @@ describe('FileDocStore', () => { expect(stream[stream.length - 1].message.s).toBe('1') }) + it('tags an agent-streamed frame so a peer tailer applies it as REDIS_AGENT_ORIGIN (never persisted)', async () => { + const streamKey = `filedoc:stream:${NAME}` + const a = await newStore() + const b = await newStore() + const bDoc = new Y.Doc() + // Capture the origin the tailer stamps each applied entry with — the persistence gate keys off it. + const origins: unknown[] = [] + bDoc.on('update', (_u: Uint8Array, origin: unknown) => origins.push(origin)) + await b.attachRoom(NAME, bDoc) + + // A normal edit tails as REDIS_ORIGIN (a peer edit that CAN be persisted). + a.publish(NAME, updateFor('user edit')) + await vi.waitFor(() => expect(origins).toContain(REDIS_ORIGIN), { timeout: 2000 }) + + // An agent-streamed frame is published WITH the agent flag: the stream entry carries the marker, and + // the peer tailer applies it as REDIS_AGENT_ORIGIN — excluded from the relay's edited/persist gate. + a.publish(NAME, updateFor('agent frame'), true) + await vi.waitFor(() => expect(origins).toContain(REDIS_AGENT_ORIGIN), { timeout: 2000 }) + const stream = state.backing!.streams.get(streamKey)! + expect(stream.some((e) => e.message.a === '1')).toBe(true) + // The normal edit's entry carries no agent marker. + expect(stream.filter((e) => e.message.a === '1')).toHaveLength(1) + bDoc.destroy() + }) + it('retries a transient append failure so the edit is not lost from the shared log', async () => { const a = await newStore() state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index b6cf65ce9e5..3b213dfb7bb 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -90,6 +90,16 @@ export const REDIS_ORIGIN = Symbol('file-doc-redis') */ export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') +/** + * Origin for an AGENT-STREAMED frame applied from the stream (a copilot output token relayed via + * {@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST}). A peer task tails these to stay live mid-stream, but + * they are transient preview content the copilot's durable `edit_content` write reconciles — so the + * relay's edit-tracker must NOT mark the doc edited on them (a startup-race duplicate between two stream + * leaders would otherwise become eligible for a peer task's persist). Behaves like {@link REDIS_ORIGIN} + * otherwise (already in the stream — never re-published). + */ +export const REDIS_AGENT_ORIGIN = Symbol('file-doc-redis-agent') + const STREAM_PREFIX = 'filedoc:stream:' /** Cluster-wide "durable version the live doc is synced to" (the persist If-Match token). */ const SYNC_VERSION_PREFIX = 'filedoc:syncver:' @@ -103,6 +113,9 @@ const UPDATE_FIELD = 'u' /** Marks a stream entry as a compaction SNAPSHOT (folds seed + edits), so the tailer applies it with * {@link REDIS_SNAPSHOT_ORIGIN}. Present only on snapshot entries. */ const SNAPSHOT_FIELD = 's' +/** Marks a stream entry as an AGENT-STREAMED preview frame, so the tailer applies it with + * {@link REDIS_AGENT_ORIGIN} (never marks the doc edited). Present only on agent-frame entries. */ +const AGENT_FIELD = 'a' /** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it @@ -264,12 +277,14 @@ export class FileDocStore { * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. */ - private async appendUpdate(name: string, update: Uint8Array): Promise { + private async appendUpdate(name: string, update: Uint8Array, agent = false): Promise { if (!this.write) return const encoded = Buffer.from(update).toString('base64') + const fields: Record = { [UPDATE_FIELD]: encoded } + if (agent) fields[AGENT_FIELD] = '1' for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { - await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: encoded }) + await this.write.xAdd(streamKey(name), '*', fields) break } catch (error) { if (attempt === PUBLISH_MAX_RETRIES) { @@ -288,11 +303,12 @@ export class FileDocStore { /** * Fire-and-forget append for the hot keystroke path (`doc.on('update')`): converges peers without - * blocking the relay. Retries internally; never throws. No-op when disabled. + * blocking the relay. Retries internally; never throws. No-op when disabled. Pass `agent: true` for + * a copilot preview frame so peer tasks tail it as {@link REDIS_AGENT_ORIGIN} and never persist it. */ - publish(name: string, update: Uint8Array): void { + publish(name: string, update: Uint8Array, agent = false): void { if (!this.enabled || !this.write) return - void this.appendUpdate(name, update).catch(() => {}) // already logged inside appendUpdate + void this.appendUpdate(name, update, agent).catch(() => {}) // already logged inside appendUpdate } /** @@ -515,8 +531,13 @@ export class FileDocStore { private applyEntry(room: StoreRoom, id: string, message: Record): void { room.lastId = id // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker - // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). - const origin = message[SNAPSHOT_FIELD] ? REDIS_SNAPSHOT_ORIGIN : REDIS_ORIGIN + // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). An + // agent-streamed preview frame is stamped separately so the tracker NEVER marks it edited. + const origin = message[SNAPSHOT_FIELD] + ? REDIS_SNAPSHOT_ORIGIN + : message[AGENT_FIELD] + ? REDIS_AGENT_ORIGIN + : REDIS_ORIGIN applyEntryToDoc(room.doc, id, message, origin) } diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 74b03b83f0e..0b96ef667c2 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -45,7 +45,12 @@ import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' -import { getFileDocStore, REDIS_ORIGIN, REDIS_SNAPSHOT_ORIGIN } from '@/handlers/file-doc-store' +import { + getFileDocStore, + REDIS_AGENT_ORIGIN, + REDIS_ORIGIN, + REDIS_SNAPSHOT_ORIGIN, +} from '@/handlers/file-doc-store' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -708,23 +713,29 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { origin === AGENT_SYNC_ORIGIN ? null : originSocketId(origin) ) // Share every locally-originated update to the stream so peers converge. Skip updates that already - // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN) and SEED_ORIGIN — the seed is published - // EXPLICITLY and AWAITED under the seed lock (so it lands before the lock releases), which a - // fire-and-forget publish here couldn't guarantee. - if (origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== SEED_ORIGIN) - getFileDocStore().publish(name, update) + // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN / REDIS_AGENT_ORIGIN) and SEED_ORIGIN — + // the seed is published EXPLICITLY and AWAITED under the seed lock (so it lands before the lock + // releases), which a fire-and-forget publish here couldn't guarantee. An agent-streamed frame + // (AGENT_SYNC_ORIGIN) is published WITH the agent marker so peer tasks tail it as REDIS_AGENT_ORIGIN + // and never mark the doc edited on it (see the edit-tracker below). + if ( + origin !== REDIS_ORIGIN && + origin !== REDIS_SNAPSHOT_ORIGIN && + origin !== REDIS_AGENT_ORIGIN && + origin !== SEED_ORIGIN + ) + getFileDocStore().publish(name, update, origin === AGENT_SYNC_ORIGIN) // Edit tracking for persistence. Mark the doc dirty on any update applied AFTER it was seeded — a // local user edit (socket origin) OR a peer's edit relayed via the tailer (REDIS_ORIGIN) — so // whichever task is last to leave persists real edits, even one that only tailed them. A compaction // snapshot on catch-up (REDIS_SNAPSHOT_ORIGIN) also counts: it folds real edits into one frame, so a // fresh task catching up purely from it must not treat the doc as unedited. The seed transition - // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. NOTE: - // an agent-streamed frame ({@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST}) is not counted on the - // ORIGINATING task (it applies under {@link AGENT_SYNC_ORIGIN} — see the handler), but in the multi-replica - // path it is published to the stream and PEER tasks apply it as REDIS_ORIGIN, indistinguishable from a - // peer edit, so it marks `edited` there. That only ever causes an extra idempotent persist of content - // the copilot's final `edit_content` write persists durably anyway (safe over-persist, never a lost - // edit); fully suppressing it would require tagging the stream entry as no-persist across replicas. + // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. An + // agent-streamed frame ({@link FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST}) is never counted anywhere: on + // the ORIGINATING task it applies under {@link AGENT_SYNC_ORIGIN}, and across replicas it is published + // WITH the agent marker so PEER tasks tail it as REDIS_AGENT_ORIGIN — neither is in the edited set + // below. So a transient startup-race duplicate between two stream leaders is never eligible for + // persistence; the copilot's durable `edit_content` write remains the sole authority over file bytes. const seededBefore = room.seededObserved if (isDocSeeded(room.doc)) room.seededObserved = true if ( From 96fd287bcedbef7552d872501bc78c23721e9fda Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:30:17 -0700 Subject: [PATCH 13/15] fix(files): reseed agent shadow on lead regain + agent-only compaction Two multi-writer edge cases surfaced in review: - rich-markdown-editor: a client that led, lost leadership, then regained it reused its stale shadow (which never saw the interim leader's ops), re-emitting ops for content already present. Tear the shadow down when a client observes it is not the leader, so a regain rebuilds fresh from the current doc. - file-doc-store: compaction always stamped its snapshot REDIS_SNAPSHOT_ORIGIN (marks peers edited). A long agent-only stream crossing the threshold could fold preview content into a persist-eligible snapshot. Track whether a room integrated any real edit and stamp an agent-only snapshot REDIS_AGENT_ORIGIN so it stays no-persist. Both covered by falsification-verified tests. --- .../src/handlers/file-doc-store.test.ts | 62 +++++++++++++++++-- apps/realtime/src/handlers/file-doc-store.ts | 49 +++++++++++++-- .../apply-streamed-markdown.test.ts | 40 ++++++++++++ .../rich-markdown-editor.tsx | 15 ++++- 4 files changed, 155 insertions(+), 11 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 9f0bf428eca..9559db74b0f 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -223,8 +223,14 @@ describe('FileDocStore', () => { const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the - // two peer entries. Inject that lagging room directly. - ;(a as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + // two peer entries. Inject that lagging room directly (a real edit was integrated → realEdited). + ;(a as any).rooms.set(NAME, { + doc: new Y.Doc(), + lastId: '400-0', + publishes: 0, + seededObserved: true, + realEdited: true, + }) await (a as any).maybeCompact(NAME) // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. @@ -264,6 +270,42 @@ describe('FileDocStore', () => { bDoc.destroy() }) + it('stamps a compaction snapshot of an agent-ONLY stream as an agent frame (never persisted)', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + // A doc whose content is purely agent preview (no real edit integrated) — realEdited stays false. + const agentDoc = docWithText('agent-only preview body') + const entries = Array.from({ length: 400 }, (_, i) => ({ + id: `${i + 1}-0`, + message: { u: noop }, + })) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 400 + + const a = await newStore() + ;(a as any).rooms.set(NAME, { + doc: agentDoc, + lastId: '400-0', + publishes: 0, + seededObserved: true, + realEdited: false, + }) + await (a as any).maybeCompact(NAME) + + // The snapshot must carry the AGENT marker, NOT the snapshot marker, so a peer catch-up applies it as + // REDIS_AGENT_ORIGIN and never marks the doc edited — the no-persist guarantee survives compaction. + const stream = state.backing!.streams.get(streamKey)! + const last = stream[stream.length - 1].message + expect(last.a).toBe('1') + expect(last.s).toBeUndefined() + // Content is still fully reconstructable from the compacted stream. + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('agent-only preview body') + doc.destroy() + agentDoc.destroy() + }) + it('retries a transient append failure so the edit is not lost from the shared log', async () => { const a = await newStore() state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed @@ -407,8 +449,20 @@ describe('FileDocStore', () => { const b = await newStore() const docA = new Y.Doc() Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 - ;(a as any).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0 }) - ;(b as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + ;(a as any).rooms.set(NAME, { + doc: docA, + lastId: '401-0', + publishes: 0, + seededObserved: true, + realEdited: true, + }) + ;(b as any).rooms.set(NAME, { + doc: new Y.Doc(), + lastId: '400-0', + publishes: 0, + seededObserved: true, + realEdited: true, + }) await Promise.all([(a as any).maybeCompact(NAME), (b as any).maybeCompact(NAME)]) const doc = new Y.Doc() diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 3b213dfb7bb..23fbf0ec77d 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -33,7 +33,7 @@ * @module */ import { createLogger } from '@sim/logger' -import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' @@ -180,6 +180,12 @@ function applyEntryToDoc( } } +/** Whether a doc carries the seed flag (mirrors the relay's `isDocSeeded`), so the store can tell the + * one-time seed transition from a real post-seed edit without re-implementing the check divergently. */ +function isDocSeeded(doc: Y.Doc): boolean { + return doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true +} + /** One locally-open room the store tracks: its doc and the last stream id applied to it. */ interface StoreRoom { doc: Y.Doc @@ -187,6 +193,14 @@ interface StoreRoom { lastId: string /** Local publish count, to pace compaction checks. */ publishes: number + /** Set once the doc has been observed seeded, so the seed transition itself is never mistaken for an + * edit (mirrors the relay's `seededObserved`). */ + seededObserved: boolean + /** Whether the doc has integrated any REAL (non-agent, non-seed) edit. Compaction stamps its snapshot + * as an AGENT snapshot ({@link REDIS_AGENT_ORIGIN}, never persisted) until this is true, so a long + * agent-only stream that crosses the compaction threshold can't fold its preview content into a + * snapshot that marks peers edited. */ + realEdited: boolean } /** @@ -250,7 +264,13 @@ export class FileDocStore { if (!this.enabled || !this.write) return // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — // the tailer resumes from `lastId`, which the catch-up advances. - const room: StoreRoom = { doc, lastId: '0', publishes: 0 } + const room: StoreRoom = { + doc, + lastId: '0', + publishes: 0, + seededObserved: false, + realEdited: false, + } this.rooms.set(name, room) try { const entries = await this.write.xRange(streamKey(name), '-', '+') @@ -298,7 +318,13 @@ export class FileDocStore { } await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) - if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + if (room) { + // A local non-agent publish (a user edit or the awaited copilot durable merge) is a real edit; the + // seed never flows through here (it uses seedIfEmpty). Set it BEFORE the compaction check below so a + // real edit can never be folded into an agent (no-persist) snapshot due to a tail-back race. + if (!agent) room.realEdited = true + if (++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + } } /** @@ -538,7 +564,16 @@ export class FileDocStore { : message[AGENT_FIELD] ? REDIS_AGENT_ORIGIN : REDIS_ORIGIN + const seededBefore = room.seededObserved applyEntryToDoc(room.doc, id, message, origin) + if (isDocSeeded(room.doc)) room.seededObserved = true + // Track a real edit integrated from the stream so compaction knows whether its snapshot represents + // real content or agent-only preview: a real snapshot (folds real edits), or a markerless edit + // applied AFTER the doc was already seeded (the seed transition itself never counts). Agent frames + // and agent snapshots (REDIS_AGENT_ORIGIN) never count. + if (origin === REDIS_SNAPSHOT_ORIGIN || (origin === REDIS_ORIGIN && seededBefore)) { + room.realEdited = true + } } /** @@ -599,10 +634,14 @@ export class FileDocStore { // appended snapshot id instead would silently drop those un-integrated peer entries. const upTo = room.lastId const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') - // Mark it a snapshot so a fresh catch-up task treats it as edited content, not a bare seed. + // Stamp the snapshot by what it folds: a real edit → SNAPSHOT_FIELD (a fresh catch-up treats it + // as edited content, not a bare seed). An agent-ONLY stream (no real edit yet) → AGENT_FIELD, so a + // peer catching up applies it as REDIS_AGENT_ORIGIN and never marks the doc edited — preserving + // the no-persist guarantee even when a long copilot stream alone crosses the compaction threshold. + const marker = room.realEdited ? SNAPSHOT_FIELD : AGENT_FIELD await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: snapshot, - [SNAPSHOT_FIELD]: '1', + [marker]: '1', }) // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts index 493fea42334..58b25009858 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts @@ -99,6 +99,46 @@ describe('agent-stream applier', () => { expect(editor.getText()).toContain('Agent wrote this') }) + it('a shadow reused after the live doc advanced duplicates content; a fresh one does not', () => { + // The invariant behind the component's leadership-regain teardown: a shadow tracks only ITS OWN + // reconciles, so once the live doc advances under another writer, REUSING that stale shadow re-emits + // ops for content already present (duplication). Seeding a FRESH shadow from the current doc fixes it. + const stale = track(makeCollabEditor()) + const staleSession = beginAgentStream(stale.editor)! // seeded from the empty base + applyAgentStreamFrame(stale.editor, staleSession, 'Alpha paragraph.') + // Another writer advances the live doc while this shadow is NOT looking (a handoff to an interim leader). + stale.editor.commands.focus('end') + stale.editor.commands.insertContent('\n\nBeta paragraph.') + // Reusing the stale shadow (only knows "Alpha") to reconcile toward the full body re-inserts "Beta". + applyAgentStreamFrame( + stale.editor, + staleSession, + 'Alpha paragraph.\n\nBeta paragraph.\n\nGamma paragraph.' + ) + endAgentStream(staleSession) + const staleText = stale.editor.getText() + expect(staleText.match(/Beta paragraph/g)?.length).toBe(2) // duplicated — what the teardown prevents + + // Fresh shadow re-seeded from the CURRENT doc (what a regaining leader does after teardown) emits only + // the genuine delta, so no content duplicates. + const fresh = track(makeCollabEditor()) + const first = beginAgentStream(fresh.editor)! + applyAgentStreamFrame(fresh.editor, first, 'Alpha paragraph.') + fresh.editor.commands.focus('end') + fresh.editor.commands.insertContent('\n\nBeta paragraph.') + endAgentStream(first) + const regained = beginAgentStream(fresh.editor)! // re-seeded from the advanced doc + applyAgentStreamFrame( + fresh.editor, + regained, + 'Alpha paragraph.\n\nBeta paragraph.\n\nGamma paragraph.' + ) + endAgentStream(regained) + const freshText = fresh.editor.getText() + expect(freshText.match(/Beta paragraph/g)?.length).toBe(1) + expect(freshText).toContain('Gamma paragraph') + }) + it('preserves a concurrent peer edit to a region the agent snapshot does not include', () => { // This is the core "AI as a CRDT peer" guarantee: the agent relays only its OWN delta (computed // against a private shadow), never a whole-document reconcile that would revert a peer's edit. diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index d9f5d77d845..58cca8d69bf 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -918,6 +918,15 @@ export function LoadedRichMarkdownEditor({ collaboration && !isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) ) { + // Not (or no longer) the leader: discard any shadow this client holds. A shadow only tracks + // ITS OWN reconciles, so one kept across a leadership loss goes stale as the interim leader + // advances the shared doc; reusing it on a later regain would re-emit ops for content already + // present (duplication). Dropping it here means a regain rebuilds a FRESH shadow from the + // current doc via the `??=` below — upholding "a non-leader holds none." + if (agentStreamSessionRef.current) { + endAgentStream(agentStreamSessionRef.current) + agentStreamSessionRef.current = null + } streamRafRef.current = null return } @@ -931,8 +940,10 @@ export function LoadedRichMarkdownEditor({ const el = containerRef.current const pinnedToBottom = el ? el.scrollHeight - el.scrollTop - el.clientHeight < 80 : false // Open the shadow lazily HERE — only when THIS client actually leads — seeded from the CURRENT - // doc, so a handoff successor diffs against the prior leader's ops (no stale base) and a - // non-leader never builds one. Defensive: a ready collab editor always has a ySync binding. + // doc. A non-leader holds none (torn down above), so whether this client is a first-time leader + // or one REGAINING leadership, `??=` finds a null ref and rebuilds fresh from the current doc, + // already carrying the interim leader's ops (never a stale base). Defensive: a ready collab + // editor always has a ySync binding. agentStreamSessionRef.current ??= beginAgentStream(editor) const session = agentStreamSessionRef.current if (!session || !applyAgentStreamFrame(editor, session, pending)) { From 8f93e66bc98eb43d36422305c7c232b3150535e4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:47:49 -0700 Subject: [PATCH 14/15] fix(files): close realEdited data-loss race + elect a settle writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent audit surfaced two real gaps: - file-doc-store: realEdited was latched AFTER appendUpdate's awaits, but the edit already sits in room.doc synchronously. A concurrent agent-frame compaction could read realEdited=false, snapshot that real content, and stamp it a no-persist agent frame — a lost edit. Latch it synchronously (same tick as the doc mutation) before any await. Deterministic falsifiable test added. - rich-markdown-editor: at settle every tab applied the final body, and a non-leader's local microtask runs before the leader's final propagates, so both insert the tail (Yjs keeps both) -> duplicated tail. Elect a single settle writer (reliable — awareness is long converged by settle), reading leadership before clearing the announcement. Corrects the overclaiming idempotency comment and the handoff pick-up comment. Adds a y-tiptap internals upgrade-guardrail test. --- .../src/handlers/file-doc-store.test.ts | 17 +++++++ apps/realtime/src/handlers/file-doc-store.ts | 19 ++++--- .../apply-streamed-markdown.test.ts | 13 +++++ .../rich-markdown-editor.tsx | 49 +++++++++++++------ 4 files changed, 75 insertions(+), 23 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 9559db74b0f..1662a8a3426 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -270,6 +270,23 @@ describe('FileDocStore', () => { bDoc.destroy() }) + it('latches realEdited synchronously so a concurrent compaction can never mislabel a real edit', async () => { + // The data-loss race: a real edit sits in room.doc synchronously, but if realEdited were set only + // AFTER appendUpdate's awaits, a concurrent agent-triggered compaction could snapshot that content and + // stamp it an agent (no-persist) frame — losing the edit. The latch must be set in the same tick. + const a = await newStore() + const doc = new Y.Doc() + await a.attachRoom(NAME, doc) + const room = (a as any).rooms.get(NAME) + expect(room.realEdited).toBe(false) + // Kick off a real (non-agent) append but do NOT await it: realEdited must already be true before the + // xAdd/expire awaits resolve, so any compaction racing on the awaits sees the real edit. + const pending = (a as any).appendUpdate(NAME, updateFor('real user edit')) + expect(room.realEdited).toBe(true) + await pending + doc.destroy() + }) + it('stamps a compaction snapshot of an agent-ONLY stream as an agent frame (never persisted)', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 23fbf0ec77d..3865fb1a6d9 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -299,6 +299,17 @@ export class FileDocStore { */ private async appendUpdate(name: string, update: Uint8Array, agent = false): Promise { if (!this.write) return + // Latch realEdited SYNCHRONOUSLY — before the first await — for a real (non-agent) publish. The edit + // already sits in room.doc (applied in doc.on('update') before publish was called), so if this set + // were deferred past the xAdd/expire awaits a CONCURRENT agent-frame-triggered maybeCompact could read + // realEdited=false, snapshot the doc (which already holds this real edit), and stamp it an agent + // (no-persist) snapshot — a lost edit. Setting it in the same synchronous tick as the doc mutation + // makes "room.doc holds a real edit ⇒ realEdited" hold before any compaction (always async) can run. + // Monotonic latch, so an eager set is safe; the seed never flows through here (it uses seedIfEmpty). + if (!agent) { + const editedRoom = this.rooms.get(name) + if (editedRoom) editedRoom.realEdited = true + } const encoded = Buffer.from(update).toString('base64') const fields: Record = { [UPDATE_FIELD]: encoded } if (agent) fields[AGENT_FIELD] = '1' @@ -318,13 +329,7 @@ export class FileDocStore { } await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) - if (room) { - // A local non-agent publish (a user edit or the awaited copilot durable merge) is a real edit; the - // seed never flows through here (it uses seedIfEmpty). Set it BEFORE the compaction check below so a - // real edit can never be folded into an agent (no-persist) snapshot due to a tail-back race. - if (!agent) room.realEdited = true - if (++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) - } + if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) } /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts index 58b25009858..8b4affc0843 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown.test.ts @@ -2,6 +2,7 @@ * @vitest-environment jsdom */ import { Editor } from '@tiptap/core' +import { initProseMirrorDoc, updateYFragment, ySyncPluginKey } from '@tiptap/y-tiptap' import { afterEach, beforeAll, describe, expect, it } from 'vitest' import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' @@ -49,6 +50,18 @@ function track(t: { editor: Editor; doc: Y.Doc; awareness: Awareness }) { } describe('agent-stream applier', () => { + it('relies on y-tiptap internals that still exist (upgrade guardrail)', () => { + // beginAgentStream/applyAgentStreamFrame reach into y-tiptap internals (not public TipTap API): + // `ySyncPluginKey`, `updateYFragment`, `initProseMirrorDoc`. A y-tiptap bump that renames or drops + // any of them can pass typecheck yet break at runtime — assert their runtime shape here so an upgrade + // fails loudly at test time instead of in production. Pinned to an exact y-tiptap version in + // package.json; bump that pin and this guard together. + expect(typeof updateYFragment).toBe('function') + expect(typeof initProseMirrorDoc).toBe('function') + expect(ySyncPluginKey).toBeDefined() + expect(typeof ySyncPluginKey.getState).toBe('function') + }) + it('streams agent content into the live collaborative doc and broadcasts it as Yjs ops', () => { const { editor, doc } = track(makeCollabEditor()) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 58cca8d69bf..a5ce37498ad 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -908,7 +908,10 @@ export function LoadedRichMarkdownEditor({ // Single-writer election: only the leader (min clientID among clients announcing they apply this // stream) writes it into the shared doc, so multiple tabs/windows watching the same live copilot // stream don't each insert it and duplicate content. A non-leader renders the leader's ops via - // Yjs; re-checked each frame, so it converges to one writer the moment awareness propagates. + // Yjs; re-checked each frame, so a co-leader stops the moment awareness propagates. (The pick-up + // direction — a successor beginning to write after the leader tab closes — waits for the next + // content frame to run a tick; a stream that already delivered its last frame is covered by + // settle and the durable write, so at worst a brief end-of-stream display lag, never a loss.) // Bounded residual (accepted): if two tabs start the SAME stream within the awareness-propagation // window they briefly both lead and duplicate a frame or two — a rare, transient, never-persisted // glitch (SYNC_NO_PERSIST keeps it out of storage; the durable edit_content write reconciles the @@ -962,27 +965,41 @@ export function LoadedRichMarkdownEditor({ cancelAnimationFrame(streamRafRef.current) streamRafRef.current = null } - // Settle: apply the FINAL body so the Y.Doc exactly equals the streamed result. The mid-stream - // leader REUSES its up-to-date shadow (just catching a throttled last frame); a client that never - // applied mid-stream (a non-leader, a held `update`, or a pre-seed stream) opens a FRESH shadow - // seeded from the CURRENT doc. Reconciling current→final is idempotent — a client that settles after - // another already wrote the final reconciles to a noop — so there is NO settle-time election and no - // base-shadow duplication, and a lone client (incl. an `update`) still applies rather than waiting on - // the durable merge. That durable `edit_content` write then lands as a noop diff too. + // Settle: apply the FINAL body so the Y.Doc exactly equals the streamed result — but ONLY the + // elected writer applies it (the same min-clientID election the streaming tick uses). Without this, + // N tabs watching one run each open a fresh shadow and reconcile current→final; a non-leader's local + // settle microtask runs BEFORE the leader's final propagates (a server round-trip), so both insert + // the same tail and Yjs keeps both (it does not dedupe identical text from two clients) → a + // duplicated tail. The election is reliable here (unlike the bounded startup window): the stream ran + // for seconds, so awareness is long converged. Each tab reads leadership BEFORE clearing its own + // announcement — a remote clear is a network round-trip, always slower than these local microtasks, + // so every tab sees the same announcer set and agrees on one leader. The leader reuses its + // up-to-date shadow (catching a throttled last frame) or, if it never applied mid-stream (a held + // `update`, or a pre-seed stream), opens a FRESH shadow from the current doc; a non-leader applies + // nothing (the leader's final broadcasts to it) and frees any shadow it still held. The durable + // `edit_content` write then lands as a noop diff for everyone. if (wasStreamingRef.current && collabReady) { wasStreamingRef.current = false agentAnnouncedRef.current = false + const isSettleWriter = + !collaboration || isAgentStreamLeader(collaboration.awareness, collaboration.doc.clientID) if (collaboration) clearAgentApplying(collaboration.awareness) lastStreamedBodyRef.current = null - const finalBody = splitFrontmatter(content).body - const session = agentStreamSessionRef.current ?? beginAgentStream(editor) + const heldSession = agentStreamSessionRef.current agentStreamSessionRef.current = null - if (session) { - runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) - // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream - // can supersede the run token and drop the apply above, but the shadow must always be - // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. - queueMicrotask(() => endAgentStream(session)) + if (isSettleWriter) { + const finalBody = splitFrontmatter(content).body + const session = heldSession ?? beginAgentStream(editor) + if (session) { + runOffRender(() => applyAgentStreamFrame(editor, session, finalBody)) + // Free the shadow with an UNGUARDED microtask (not `runOffRender`): a rapid follow-up stream + // can supersede the run token and drop the apply above, but the shadow must always be + // destroyed. Queued after the apply, so it frees the shadow only once that has had its chance. + queueMicrotask(() => endAgentStream(session)) + } + } else if (heldSession) { + // Non-leader: it never writes the final (the leader does + broadcasts it); free any shadow it held. + queueMicrotask(() => endAgentStream(heldSession)) } } return From ca81494072acec7b39ed851768e71b1cd8ed5fe1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:12:19 -0700 Subject: [PATCH 15/15] fix(files): own presence per client id, not one-per-socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared workspace socket hosts one collaborative provider per mounted view, so the chat file preview and the standalone Files editor for the same file each bind their own Yjs client id over ONE socket. The relay owned a single client id per socket, so the later JOIN overwrote the earlier and dropped its awareness — which silently broke the single-writer agent-stream election (a peer stopped seeing the streaming provider's announcement and could self-elect, duplicating streamed text for the whole stream). Track ownership per (socket, client id): a socket owns a set of client ids; the awareness gate accepts a frame only if every id it carries is owned; cleanup drops all of a socket's ids; the roster stays one-entry-per-session. Reclaim and the same-user reconnect path evict just the reclaimed id, dropping the old socket only if it empties. Falsification-verified test added. --- apps/realtime/src/handlers/file-doc.test.ts | 36 +++++--- apps/realtime/src/handlers/file-doc.ts | 99 +++++++++++++-------- 2 files changed, 86 insertions(+), 49 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 70f5351860c..5121638962a 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -713,23 +713,33 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) }) - it('clears a departed caret when a socket rejoins the room with a new client id', async () => { + it('a socket owns MULTIPLE client ids (co-mounted providers) and relays awareness for each', async () => { + // The shared workspace socket hosts one provider per collaborative view, so the chat file preview + // and the standalone Files editor for the same file each JOIN with their own Yjs client id over ONE + // socket. Ownership is per client id: BOTH announcements must relay. (The old one-owner-per-socket + // model let the later JOIN overwrite the earlier, dropping its awareness — which broke the + // single-writer agent-stream election, letting a peer also self-elect and duplicate streamed text.) const { io, sent } = createIo() - const { frame: awFrame } = awarenessFrame(500, 'A') const a = setup('socket-a', io) await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 500 }) - a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) - sent.length = 0 - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 501 }) - - // The old client (500) caret removal is broadcast to the room. - const removal = sent.find( - (m) => - m.event === FILE_DOC_EVENTS.MESSAGE && - (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS - ) - expect(removal).toBeDefined() + expect(joinSuccessFileId(a.socket)).toBe('file-1') + + const relayedFor = (clientId: number) => { + sent.length = 0 + a.handlers[FILE_DOC_EVENTS.MESSAGE](awarenessFrame(clientId, `c${clientId}`).frame) + return sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + } + // The FIRST provider's client id (500) is still owned after the second joins — its awareness relays. + expect(relayedFor(500)).toBeDefined() + // The second provider's client id (501) relays too. + expect(relayedFor(501)).toBeDefined() + // A client id this socket does NOT own is still dropped (ownership is not blanket-allowed). + expect(relayedFor(999)).toBeUndefined() }) it('preserves the existing caret when a rebind to a foreign client id is rejected', async () => { diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 0b96ef667c2..a907cbb9cda 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -90,12 +90,16 @@ const MERGE_LOCK_RETRIES = Math.ceil( (MERGE_LOCK_TTL_MS + FILE_DOC_TIMEOUTS.mergeRequestMs) / MERGE_LOCK_RETRY_MS ) -/** A socket's presence ownership within a room. */ +/** One presence ownership within a room: a (socket, clientID) pair. */ interface FileDocOwner { /** - * The awareness clientID the socket declared at join. It owns exactly this one - * and may only publish/remove awareness for it, so an authenticated peer cannot - * forge or clear another collaborator's presence. + * An awareness clientID this socket declared at join. The socket may only publish/remove awareness + * for a clientID it owns, so an authenticated peer cannot forge or clear another collaborator's + * presence. A single socket can own SEVERAL clientIDs at once — the shared workspace socket hosts one + * provider per mounted collaborative view, so e.g. the chat file preview and the standalone Files + * editor for the same file each bind their own Yjs clientID over the one socket. The election that + * picks a single agent-stream writer depends on every such provider's awareness propagating, so + * ownership is tracked per clientID, not one-per-socket (which would drop the later joiner's frames). */ clientId: number /** The owning user — used to tell a reconnect (same user reusing its Yjs client @@ -112,8 +116,9 @@ interface FileDocRoom { fileId: string doc: Y.Doc awareness: awarenessProtocol.Awareness - /** socketId → its presence ownership. */ - owners: Map + /** socketId → (clientId → its presence ownership). A socket owns one entry per collaborative provider + * it mounted for this file (see {@link FileDocOwner}); an empty inner map is never kept. */ + owners: Map> /** True once the server-side seed fetch has started, so concurrent joins don't each fetch. * Reset on a fetch FAILURE so a later join can retry (a genuinely empty file stays empty). */ serverSeedStarted: boolean @@ -365,7 +370,12 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr */ function broadcastFileDocPresence(io: Server, name: string, room: FileDocRoom) { const users: FileDocPresenceUser[] = [] - for (const [socketId, owner] of room.owners) { + // One entry PER SOCKET (session), not per clientID: a socket's several providers are the same + // authenticated user, so any of its owners carries the identity; the client dedupes per user for the + // avatar stack (see the roster comment above). An empty inner map is never stored, so `owner` exists. + for (const [socketId, clientMap] of room.owners) { + const owner = clientMap.values().next().value + if (!owner) continue users.push({ socketId, userId: owner.userId, @@ -799,8 +809,9 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { switch (messageType) { case FILE_DOC_MESSAGE_TYPE.SYNC: { - // Attribute a server-side persist of the resulting edit to the actual editor (blob metadata). - const editor = room.owners.get(socket.id)?.userId + // Attribute a server-side persist of the resulting edit to the actual editor (blob metadata). A + // socket's providers are all the same user, so any owner's userId identifies the editor. + const editor = room.owners.get(socket.id)?.values().next().value?.userId if (editor) room.lastEditorUserId = editor const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) @@ -831,11 +842,12 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { } case FILE_DOC_MESSAGE_TYPE.AWARENESS: { const update = decoding.readVarUint8Array(decoder) - // Enforce presence ownership: a socket may only publish/remove awareness - // for the clientID it bound at join, so a peer cannot spoof or clear - // another collaborator's caret. - const owned = room.owners.get(socket.id)?.clientId - if (owned === undefined || awarenessUpdateClientIds(update).some((id) => id !== owned)) { + // Enforce presence ownership: a socket may only publish/remove awareness for a clientID it bound + // at join, so a peer cannot spoof or clear another collaborator's caret. A socket can own SEVERAL + // clientIDs (one per mounted provider), so the frame is accepted only if EVERY id it carries is + // owned by this socket. + const owned = room.owners.get(socket.id) + if (owned === undefined || awarenessUpdateClientIds(update).some((id) => !owned.has(id))) { logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) return } @@ -873,12 +885,16 @@ export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife const room = fileDocRooms.get(name) if (!room) return - const owner = room.owners.get(socketId) + // The socket may own several clientIDs (one per provider it mounted for this file); drop them ALL. + // The client only emits LEAVE / disconnects once its LAST provider for the file tears down, so a + // per-socket cleanup here is correct — an earlier single-provider unmount already cleared its own + // caret via its awareness removal. + const clientMap = room.owners.get(socketId) room.owners.delete(socketId) - if (owner !== undefined) { - // Fires the awareness `update` handler with a non-socket origin → the removal - // is broadcast to every remaining client, so the departed caret vanishes. - awarenessProtocol.removeAwarenessStates(room.awareness, [owner.clientId], null) + if (clientMap !== undefined && clientMap.size > 0) { + // Fires the awareness `update` handler with a non-socket origin → the removals + // are broadcast to every remaining client, so the departed carets vanish. + awarenessProtocol.removeAwarenessStates(room.awareness, [...clientMap.keys()], null) // Refresh the roster for whoever remains (server-authenticated identity). broadcastFileDocPresence(io, name, room) } @@ -980,20 +996,27 @@ export function setupWorkspaceFileDocHandlers( // rejected. This runs BEFORE any teardown of the socket's current binding below, so a // rejected rebind — even during a document switch — leaves the socket's existing document // and caret untouched. - for (const [otherSid, owner] of entry.owners) { - if (owner.clientId !== clientId || otherSid === socket.id) continue + for (const [otherSid, clientMap] of entry.owners) { + if (otherSid === socket.id) continue + const owner = clientMap.get(clientId) + if (owner === undefined) continue if (owner.userId !== userId) { emitJoinError(socket, fileId, 'Client id already in use', 'CLIENT_ID_IN_USE', false) return } - // Fully evict the stale prior socket of the same user — owner + caret AND its room - // mapping + Socket.IO membership — so it can no longer send document (sync) frames: - // handleMessage's SYNC path gates on socketToRoomName, not owners. Done inline rather - // than via cleanupFileDocForSocket, which could destroyRoomIfIdle the room we're joining. - entry.owners.delete(otherSid) - awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) - socketToRoomName.delete(otherSid) - io.in(otherSid).socketsLeave(name) + // Same user reclaiming its client id on a stale prior socket: evict just THAT clientID's binding + // + caret from the old socket. If that leaves the old socket with no providers, also drop its + // room mapping + Socket.IO membership so it can no longer send document (sync) frames + // (handleMessage's SYNC path gates on socketToRoomName, not owners); an old socket that still + // hosts OTHER providers keeps them. Done inline rather than via cleanupFileDocForSocket, which + // could destroyRoomIfIdle the room we're joining. + clientMap.delete(clientId) + awarenessProtocol.removeAwarenessStates(entry.awareness, [clientId], null) + if (clientMap.size === 0) { + entry.owners.delete(otherSid) + socketToRoomName.delete(otherSid) + io.in(otherSid).socketsLeave(name) + } } // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if @@ -1005,14 +1028,18 @@ export function setupWorkspaceFileDocHandlers( cleanupFileDocForSocket(socket.id, io) } - // Accepted: a same socket rebinding to a NEW client id clears its old caret - // so it doesn't linger as a ghost after the binding is overwritten. - const previous = entry.owners.get(socket.id) - if (previous !== undefined && previous.clientId !== clientId) { - awarenessProtocol.removeAwarenessStates(entry.awareness, [previous.clientId], null) + // ADD this provider's clientID to the socket's ownership set (do NOT overwrite a sibling provider + // on the same socket — that lone-owner overwrite is exactly what dropped the chat preview's + // awareness when the Files editor co-mounted). A re-JOIN of the same clientID is idempotent. A + // single provider that later unmounts clears its own caret via its awareness removal; the whole + // set is dropped on the socket's LEAVE/disconnect (client emits LEAVE only after its LAST provider + // for the file tears down). + let clientMap = entry.owners.get(socket.id) + if (clientMap === undefined) { + clientMap = new Map() + entry.owners.set(socket.id, clientMap) } - - entry.owners.set(socket.id, { clientId, userId, userName, avatarUrl }) + clientMap.set(clientId, { clientId, userId, userName, avatarUrl }) socketToRoomName.set(socket.id, name) socket.join(name)