From 169abc62916475f8fac0034c6a227ed52ed316f0 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Fri, 8 May 2026 23:07:32 +0100 Subject: [PATCH 01/10] Skip shell summary refresh for streaming deltas --- apps/server/src/orchestration/Layers/ProjectionPipeline.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 1161ff6a7d..42a2dcbbf6 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -701,6 +701,13 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti ...existingRow.value, updatedAt: event.occurredAt, }); + if ( + event.type === "thread.message-sent" && + event.payload.role === "assistant" && + event.payload.streaming + ) { + return; + } yield* refreshThreadShellSummary(event.payload.threadId); return; } From 5d42371ad288e7c3fe1ebe309ade2c6adf46f9bc Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Fri, 8 May 2026 23:12:51 +0100 Subject: [PATCH 02/10] Append streaming assistant deltas in-place --- .../Layers/ProjectionPipeline.test.ts | 123 ++++++++++++++++++ .../Layers/ProjectionPipeline.ts | 21 +++ .../Layers/ProjectionThreadMessages.ts | 51 ++++++++ .../Services/ProjectionThreadMessages.ts | 24 ++++ 4 files changed, 219 insertions(+) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 369eea0f7a..df1fd6bc9a 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -596,6 +596,129 @@ it.layer( ); }); +it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-streaming-append-")))( + "OrchestrationProjectionPipeline", + (it) => { + it.effect("appends assistant streaming deltas in-place while preserving row metadata", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); + const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); + const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); + + const appendAndProject = (event: Parameters[0]) => + eventStore + .append(event) + .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent))); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-1"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: first, + commandId: CommandId.make("cmd-streaming-append-1"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-1"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: "Hello", + attachments: [ + { + type: "image", + id: "thread-streaming-append-att-1", + name: "first.png", + mimeType: "image/png", + sizeBytes: 5, + }, + ], + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: first, + updatedAt: first, + }, + }); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-2"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: second, + commandId: CommandId.make("cmd-streaming-append-2"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-2"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: " world", + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: second, + updatedAt: second, + }, + }); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-3"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: third, + commandId: CommandId.make("cmd-streaming-append-3"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-3"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: "!", + attachments: [], + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: third, + updatedAt: third, + }, + }); + + const rows = yield* sql<{ + readonly text: string; + readonly attachmentsJson: string | null; + readonly createdAt: string; + readonly updatedAt: string; + readonly isStreaming: number; + }>` + SELECT + text, + attachments_json AS "attachmentsJson", + created_at AS "createdAt", + updated_at AS "updatedAt", + is_streaming AS "isStreaming" + FROM projection_thread_messages + WHERE message_id = 'message-streaming-append' + `; + + assert.equal(rows.length, 1); + assert.deepEqual(rows[0], { + text: "Hello world!", + attachmentsJson: "[]", + createdAt: first, + updatedAt: third, + isStreaming: 1, + }); + }), + ); + }, +); + it.layer( Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-attachments-rollback-")), )("OrchestrationProjectionPipeline", (it) => { diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 42a2dcbbf6..0cb3ac7642 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -792,6 +792,27 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti )(function* (event, attachmentSideEffects) { switch (event.type) { case "thread.message-sent": { + if (event.payload.role === "assistant" && event.payload.streaming) { + const nextAttachments = + event.payload.attachments !== undefined + ? yield* materializeAttachmentsForProjection({ + attachments: event.payload.attachments, + }) + : undefined; + yield* projectionThreadMessageRepository.appendText({ + messageId: event.payload.messageId, + threadId: event.payload.threadId, + turnId: event.payload.turnId, + role: event.payload.role, + textDelta: event.payload.text, + ...(nextAttachments !== undefined ? { attachments: [...nextAttachments] } : {}), + isStreaming: event.payload.streaming, + createdAt: event.payload.createdAt, + updatedAt: event.payload.updatedAt, + }); + return; + } + const existingMessage = yield* projectionThreadMessageRepository.getByMessageId({ messageId: event.payload.messageId, }); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts index 7191916688..67396be6c0 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts @@ -9,6 +9,7 @@ import { ChatAttachment } from "@t3tools/contracts"; import { toPersistenceSqlError } from "../Errors.ts"; import { + AppendProjectionThreadMessageTextInput, GetProjectionThreadMessageInput, ProjectionThreadMessageRepository, type ProjectionThreadMessageRepositoryShape, @@ -95,6 +96,50 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { }, }); + const appendProjectionThreadMessageText = SqlSchema.void({ + Request: AppendProjectionThreadMessageTextInput, + execute: (row) => { + const nextAttachmentsJson = + row.attachments !== undefined ? JSON.stringify(row.attachments) : null; + return sql` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + attachments_json, + is_streaming, + created_at, + updated_at + ) + VALUES ( + ${row.messageId}, + ${row.threadId}, + ${row.turnId}, + ${row.role}, + ${row.textDelta}, + ${nextAttachmentsJson}, + ${row.isStreaming ? 1 : 0}, + ${row.createdAt}, + ${row.updatedAt} + ) + ON CONFLICT (message_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + role = excluded.role, + text = projection_thread_messages.text || excluded.text, + attachments_json = COALESCE( + excluded.attachments_json, + projection_thread_messages.attachments_json + ), + is_streaming = excluded.is_streaming, + updated_at = excluded.updated_at + `; + }, + }); + const getProjectionThreadMessageRow = SqlSchema.findOneOption({ Request: GetProjectionThreadMessageInput, Result: ProjectionThreadMessageDbRowSchema, @@ -151,6 +196,11 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.upsert:query")), ); + const appendText: ProjectionThreadMessageRepositoryShape["appendText"] = (input) => + appendProjectionThreadMessageText(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.appendText:query")), + ); + const getByMessageId: ProjectionThreadMessageRepositoryShape["getByMessageId"] = (input) => getProjectionThreadMessageRow(input).pipe( Effect.mapError( @@ -176,6 +226,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { return { upsert, + appendText, getByMessageId, listByThreadId, deleteByThreadId, diff --git a/apps/server/src/persistence/Services/ProjectionThreadMessages.ts b/apps/server/src/persistence/Services/ProjectionThreadMessages.ts index d50ff32025..f1a1bf93ee 100644 --- a/apps/server/src/persistence/Services/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Services/ProjectionThreadMessages.ts @@ -49,6 +49,20 @@ export const DeleteProjectionThreadMessagesInput = Schema.Struct({ }); export type DeleteProjectionThreadMessagesInput = typeof DeleteProjectionThreadMessagesInput.Type; +export const AppendProjectionThreadMessageTextInput = Schema.Struct({ + messageId: MessageId, + threadId: ThreadId, + turnId: Schema.NullOr(TurnId), + role: OrchestrationMessageRole, + textDelta: Schema.String, + attachments: Schema.optional(Schema.Array(ChatAttachment)), + isStreaming: Schema.Boolean, + createdAt: IsoDateTime, + updatedAt: IsoDateTime, +}); +export type AppendProjectionThreadMessageTextInput = + typeof AppendProjectionThreadMessageTextInput.Type; + /** * ProjectionThreadMessageRepositoryShape - Service API for projected thread messages. */ @@ -62,6 +76,16 @@ export interface ProjectionThreadMessageRepositoryShape { message: ProjectionThreadMessage, ) => Effect.Effect; + /** + * Insert a projected message or append a streaming text delta in-place. + * + * Upserts by `messageId`. Existing attachments are preserved unless the + * append carries replacement attachments. + */ + readonly appendText: ( + input: AppendProjectionThreadMessageTextInput, + ) => Effect.Effect; + /** * Read a projected thread message by id. */ From 5b77a779eb7e8497a601b3f2d5b1d3b198cb6f50 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Sat, 9 May 2026 18:23:36 +0100 Subject: [PATCH 03/10] Add assistant streaming replay benchmark --- .../server/scripts/replay_thread/benchmark.ts | 1570 +++++++++++++++++ .../server/scripts/replay_thread/core.test.ts | 77 + apps/server/scripts/replay_thread/core.ts | 108 ++ 3 files changed, 1755 insertions(+) create mode 100644 apps/server/scripts/replay_thread/benchmark.ts create mode 100644 apps/server/scripts/replay_thread/core.test.ts create mode 100644 apps/server/scripts/replay_thread/core.ts diff --git a/apps/server/scripts/replay_thread/benchmark.ts b/apps/server/scripts/replay_thread/benchmark.ts new file mode 100644 index 0000000000..c58c8e8454 --- /dev/null +++ b/apps/server/scripts/replay_thread/benchmark.ts @@ -0,0 +1,1570 @@ +import { existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + OrchestrationEvent, + ThreadId, + type OrchestrationEvent as OrchestrationEventType, +} from "@t3tools/contracts"; +import { Database } from "bun:sqlite"; +import { Effect, Layer, Option, Schema } from "effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { ServerConfig } from "../../src/config.ts"; +import { OrchestrationEventStoreLive } from "../../src/persistence/Layers/OrchestrationEventStore.ts"; +import { + makeSqlitePersistenceLive, + SqlitePersistenceMemory, +} from "../../src/persistence/Layers/Sqlite.ts"; +import { RepositoryIdentityResolver } from "../../src/project/Services/RepositoryIdentityResolver.ts"; +import { OrchestrationProjectionPipelineLive } from "../../src/orchestration/Layers/ProjectionPipeline.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; +import { OrchestrationProjectionPipeline } from "../../src/orchestration/Services/ProjectionPipeline.ts"; +import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import { + buildTimingSamples, + calculateTimingStats, + checksumRows, + classifyReplayEvent, +} from "./core.ts"; + +interface CliOptions { + readonly sourceDb: string; + readonly threadId: string; + readonly target: "memory" | "file"; + readonly targetFile: string | null; + readonly verify: "none" | "messages" | "full"; + readonly diffMessages: number; + readonly diffOnly: boolean; + readonly compare: "none" | "assistant-streaming"; + readonly legacyMode: "sampled" | "full"; + readonly legacySamplePerWindow: number; + readonly windowSize: number; + readonly sampleEvery: number; + readonly progressEvery: number; + readonly limit: number | null; + readonly keepTarget: boolean; +} + +interface SourceEventRow { + readonly sequence: number; + readonly eventId: string; + readonly aggregateKind: "project" | "thread"; + readonly aggregateId: string; + readonly type: string; + readonly occurredAt: string; + readonly commandId: string | null; + readonly causationEventId: string | null; + readonly correlationId: string | null; + readonly payload: string; + readonly metadata: string; +} + +interface SourceProjectRow { + readonly projectId: string; + readonly title: string; + readonly workspaceRoot: string; + readonly scriptsJson: string; + readonly createdAt: string; + readonly updatedAt: string; + readonly deletedAt: string | null; + readonly defaultModelSelectionJson: string | null; +} + +interface SourceMessageRow { + readonly messageId: string; + readonly threadId: string; + readonly turnId: string | null; + readonly role: string; + readonly text: string; + readonly attachmentsJson: string | null; + readonly isStreaming: number; + readonly createdAt: string; + readonly updatedAt: string; +} + +interface AssistantStreamingEventRow { + readonly messageId: string; + readonly threadId: string; + readonly turnId: string | null; + readonly role: string; + readonly text: string; + readonly createdAt: string; + readonly updatedAt: string; +} + +type SqlBindingRecord = Record; + +const decodeEvent = Schema.decodeUnknownSync(OrchestrationEvent); + +function parseArgs(argv: ReadonlyArray): CliOptions { + let sourceDb: string | undefined; + let threadId: string | undefined; + let target: CliOptions["target"] = "memory"; + let targetFile: string | null = null; + let verify: CliOptions["verify"] = "messages"; + let diffMessages = 0; + let diffOnly = false; + let compare: CliOptions["compare"] = "none"; + let legacyMode: CliOptions["legacyMode"] = "sampled"; + let legacySamplePerWindow = 500; + let windowSize = 10_000; + let sampleEvery = 1_000; + let progressEvery = 10_000; + let limit: number | null = null; + let keepTarget = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => { + const value = argv[index + 1]; + if (value === undefined) { + throw new Error(`Missing value for ${arg}`); + } + index += 1; + return value; + }; + + switch (arg) { + case "--source-db": + sourceDb = next(); + break; + case "--thread-id": + threadId = next(); + break; + case "--target": { + const value = next(); + if (value !== "memory" && value !== "file") { + throw new Error("--target must be memory or file"); + } + target = value; + break; + } + case "--target-file": + targetFile = next(); + target = "file"; + break; + case "--verify": { + const value = next(); + if (value !== "none" && value !== "messages" && value !== "full") { + throw new Error("--verify must be none, messages, or full"); + } + verify = value; + break; + } + case "--diff-messages": + diffMessages = Number(next()); + break; + case "--diff-only": + diffOnly = true; + break; + case "--compare": { + const value = next(); + if (value !== "assistant-streaming") { + throw new Error("--compare must be assistant-streaming"); + } + compare = value; + break; + } + case "--legacy-mode": { + const value = next(); + if (value !== "sampled" && value !== "full") { + throw new Error("--legacy-mode must be sampled or full"); + } + legacyMode = value; + break; + } + case "--legacy-sample-per-window": + legacySamplePerWindow = Number(next()); + break; + case "--window-size": + windowSize = Number(next()); + break; + case "--sample-every": + sampleEvery = Number(next()); + break; + case "--progress-every": + progressEvery = Number(next()); + break; + case "--limit": + limit = Number(next()); + break; + case "--keep-target": + keepTarget = true; + break; + case "--help": + case "-h": + printUsage(); + process.exit(0); + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!sourceDb) { + throw new Error("Missing --source-db"); + } + if (!threadId) { + throw new Error("Missing --thread-id"); + } + if (!Number.isFinite(sampleEvery) || sampleEvery < 1) { + throw new Error("--sample-every must be a positive number"); + } + if (!Number.isFinite(diffMessages) || diffMessages < 0) { + throw new Error("--diff-messages must be zero or a positive number"); + } + if (!Number.isFinite(progressEvery) || progressEvery < 1) { + throw new Error("--progress-every must be a positive number"); + } + if (!Number.isFinite(legacySamplePerWindow) || legacySamplePerWindow < 1) { + throw new Error("--legacy-sample-per-window must be a positive number"); + } + if (!Number.isFinite(windowSize) || windowSize < 1) { + throw new Error("--window-size must be a positive number"); + } + if (limit !== null && (!Number.isFinite(limit) || limit < 1)) { + throw new Error("--limit must be a positive number"); + } + + return { + sourceDb: resolve(sourceDb), + threadId, + target, + targetFile, + verify, + diffMessages: Math.floor(diffMessages), + diffOnly, + compare, + legacyMode, + legacySamplePerWindow: Math.floor(legacySamplePerWindow), + windowSize: Math.floor(windowSize), + sampleEvery: Math.floor(sampleEvery), + progressEvery: Math.floor(progressEvery), + limit: limit === null ? null : Math.floor(limit), + keepTarget, + }; +} + +function printUsage() { + console.log(`Usage: + bun apps/server/scripts/replay_thread/benchmark.ts \\ + --source-db C:\\Users\\mike\\.t3\\dev\\state.sqlite \\ + --thread-id de1c398f-5d3c-40e4-911c-2b672653cda7 + +Options: + --target memory|file Replay into an isolated in-memory DB by default. + --target-file Replay into a temp/inspectable SQLite file. + --verify none|messages|full + Verify messages by default. Full checks every table. + --diff-messages Print the first n source-vs-event message diffs. + --diff-only Only run message diffing; skip projection replay. + --compare assistant-streaming + Compare optimized vs legacy assistant streaming paths. + --legacy-mode sampled|full + Default: sampled. Full can take hours on large threads. + --legacy-sample-per-window + Legacy samples per window in sampled mode. Default: 500. + --window-size Assistant-streaming compare window size. Default: 10000. + --sample-every Print timing windows of n events. Default: 1000. + --progress-every Print progress every n events. Default: 10000. + --limit Replay only the first n events for smoke testing. + --keep-target Keep target file when --target-file is used. +`); +} + +function readSourceEvents( + sourceDb: string, + threadId: string, +): ReadonlyArray { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + const rows = db + .query(` + SELECT + sequence, + event_id AS eventId, + aggregate_kind AS aggregateKind, + stream_id AS aggregateId, + event_type AS type, + occurred_at AS occurredAt, + command_id AS commandId, + causation_event_id AS causationEventId, + correlation_id AS correlationId, + payload_json AS payload, + metadata_json AS metadata + FROM orchestration_events + WHERE aggregate_kind = 'thread' + AND stream_id = ? + ORDER BY sequence ASC + `) + .all(threadId); + return rows.map((row) => + decodeEvent({ + ...row, + payload: JSON.parse(row.payload), + metadata: JSON.parse(row.metadata), + }), + ); + } finally { + db.close(); + } +} + +function readSourceProject(sourceDb: string, threadId: string): SourceProjectRow | null { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + return ( + db + .query(` + SELECT + p.project_id AS projectId, + p.title, + p.workspace_root AS workspaceRoot, + p.scripts_json AS scriptsJson, + p.created_at AS createdAt, + p.updated_at AS updatedAt, + p.deleted_at AS deletedAt, + p.default_model_selection_json AS defaultModelSelectionJson + FROM projection_threads t + JOIN projection_projects p ON p.project_id = t.project_id + WHERE t.thread_id = ? + LIMIT 1 + `) + .get(threadId) ?? null + ); + } finally { + db.close(); + } +} + +function readAssistantStreamingEvents( + sourceDb: string, + threadId: string, +): ReadonlyArray { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + return db + .query(` + SELECT + json_extract(payload_json, '$.messageId') AS messageId, + json_extract(payload_json, '$.threadId') AS threadId, + json_extract(payload_json, '$.turnId') AS turnId, + json_extract(payload_json, '$.role') AS role, + json_extract(payload_json, '$.text') AS text, + json_extract(payload_json, '$.createdAt') AS createdAt, + json_extract(payload_json, '$.updatedAt') AS updatedAt + FROM orchestration_events + WHERE aggregate_kind = 'thread' + AND stream_id = ? + AND event_type = 'thread.message-sent' + AND json_extract(payload_json, '$.role') = 'assistant' + AND json_extract(payload_json, '$.streaming') = 1 + ORDER BY sequence ASC + `) + .all(threadId); + } finally { + db.close(); + } +} + +function readSourceThreadRow(sourceDb: string, threadId: string): Record { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + const row = db + .query, [string]>(` + SELECT + thread_id, + project_id, + title, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at, + runtime_mode, + interaction_mode, + model_selection_json, + archived_at, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan + FROM projection_threads + WHERE thread_id = ? + `) + .get(threadId); + if (row === null) { + throw new Error(`Thread ${threadId} not found in source DB`); + } + return row; + } finally { + db.close(); + } +} + +function readSourceActivityRows( + sourceDb: string, + threadId: string, +): ReadonlyArray> { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + return db + .query, [string]>(` + SELECT + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + created_at, + sequence + FROM projection_thread_activities + WHERE thread_id = ? + `) + .all(threadId); + } finally { + db.close(); + } +} + +function readSourceChecksumRows( + sourceDb: string, + table: string, + threadId: string, +): ReadonlyArray> { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + return db.query, [string]>(checksumQuery(table)).all(threadId); + } finally { + db.close(); + } +} + +function readSourceMessageRows( + sourceDb: string, + threadId: string, +): ReadonlyArray { + const db = new Database(sourceDb, { readonly: true, strict: true }); + try { + return db + .query(` + SELECT + message_id AS messageId, + thread_id AS threadId, + turn_id AS turnId, + role, + text, + attachments_json AS attachmentsJson, + is_streaming AS isStreaming, + created_at AS createdAt, + updated_at AS updatedAt + FROM projection_thread_messages + WHERE thread_id = ? + ORDER BY created_at, message_id + `) + .all(threadId); + } finally { + db.close(); + } +} + +function stringifyAttachments(attachments: unknown): string | null { + return attachments === undefined ? null : JSON.stringify(attachments); +} + +function deriveMessageRowsFromEvents( + events: ReadonlyArray, +): ReadonlyArray { + const messages = new Map(); + for (const event of events) { + if (event.type !== "thread.message-sent") { + continue; + } + const previous = messages.get(event.payload.messageId); + const text = + previous === undefined + ? event.payload.text + : event.payload.streaming + ? `${previous.text}${event.payload.text}` + : event.payload.text.length === 0 + ? previous.text + : event.payload.text; + const attachments = + event.payload.attachments !== undefined + ? event.payload.attachments + : previous?.attachmentsJson !== null && previous?.attachmentsJson !== undefined + ? JSON.parse(previous.attachmentsJson) + : undefined; + messages.set(event.payload.messageId, { + messageId: event.payload.messageId, + threadId: event.payload.threadId, + turnId: event.payload.turnId, + role: event.payload.role, + text, + attachmentsJson: stringifyAttachments(attachments), + isStreaming: event.payload.streaming ? 1 : 0, + createdAt: previous?.createdAt ?? event.payload.createdAt, + updatedAt: event.payload.updatedAt, + }); + } + return [...messages.values()].toSorted( + (left, right) => + left.createdAt.localeCompare(right.createdAt) || + left.messageId.localeCompare(right.messageId), + ); +} + +function compactValue(value: unknown): unknown { + if (typeof value === "string" && value.length > 180) { + return `${value.slice(0, 180)}...`; + } + return value; +} + +function buildMessageDiffs(input: { + readonly sourceRows: ReadonlyArray; + readonly expectedRows: ReadonlyArray; + readonly limit: number; +}): ReadonlyArray> { + if (input.limit <= 0) { + return []; + } + const sourceById = new Map(input.sourceRows.map((row) => [row.messageId, row] as const)); + const expectedById = new Map(input.expectedRows.map((row) => [row.messageId, row] as const)); + const diffs: Array> = []; + const keys = [ + "threadId", + "turnId", + "role", + "text", + "attachmentsJson", + "isStreaming", + "createdAt", + "updatedAt", + ] as const; + + for (const [messageId, expected] of expectedById) { + const source = sourceById.get(messageId); + if (source === undefined) { + diffs.push({ messageId, kind: "missing-source" }); + } else { + for (const key of keys) { + if (source[key] !== expected[key]) { + const doubled = + key === "text" && + typeof source.text === "string" && + source.text.length === expected.text.length * 2 && + source.text === `${expected.text}${expected.text}`; + diffs.push({ + messageId, + key, + source: compactValue(source[key]), + expected: compactValue(expected[key]), + sourceLength: typeof source[key] === "string" ? source[key].length : undefined, + expectedLength: typeof expected[key] === "string" ? expected[key].length : undefined, + ...(doubled ? { looksDoubled: true } : {}), + }); + break; + } + } + } + if (diffs.length >= input.limit) { + return diffs; + } + } + + for (const messageId of sourceById.keys()) { + if (!expectedById.has(messageId)) { + diffs.push({ messageId, kind: "extra-source" }); + } + if (diffs.length >= input.limit) { + return diffs; + } + } + + return diffs; +} + +function checksumQuery(table: string): string { + switch (table) { + case "projection_threads": + return ` + SELECT + thread_id, + project_id, + title, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at, + runtime_mode, + interaction_mode, + model_selection_json, + archived_at, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan + FROM projection_threads + WHERE thread_id = ? + ORDER BY thread_id + `; + case "projection_thread_messages": + return ` + SELECT * + FROM projection_thread_messages + WHERE thread_id = ? + ORDER BY created_at, message_id + `; + case "projection_thread_activities": + return ` + SELECT * + FROM projection_thread_activities + WHERE thread_id = ? + ORDER BY sequence, created_at, activity_id + `; + case "projection_turns": + return ` + SELECT + thread_id, + turn_id, + pending_message_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_turn_count, + checkpoint_ref, + checkpoint_status, + checkpoint_files_json, + source_proposed_plan_thread_id, + source_proposed_plan_id + FROM projection_turns + WHERE thread_id = ? + ORDER BY requested_at, row_id + `; + case "projection_thread_sessions": + return ` + SELECT * + FROM projection_thread_sessions + WHERE thread_id = ? + ORDER BY thread_id + `; + case "projection_thread_proposed_plans": + return ` + SELECT * + FROM projection_thread_proposed_plans + WHERE thread_id = ? + ORDER BY created_at, plan_id + `; + case "projection_pending_approvals": + return ` + SELECT * + FROM projection_pending_approvals + WHERE thread_id = ? + ORDER BY created_at, request_id + `; + default: + throw new Error(`Unsupported checksum table: ${table}`); + } +} + +const readTargetChecksumRows = ( + sql: SqlClient.SqlClient, + table: (typeof checksumTables)[number], + threadId: string, +): Effect.Effect>, Error> => { + const rows = (() => { + switch (table) { + case "projection_threads": + return sql>` + SELECT + thread_id, + project_id, + title, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at, + runtime_mode, + interaction_mode, + model_selection_json, + archived_at, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan + FROM projection_threads + WHERE thread_id = ${threadId} + ORDER BY thread_id + `; + case "projection_thread_messages": + return sql>` + SELECT * + FROM projection_thread_messages + WHERE thread_id = ${threadId} + ORDER BY created_at, message_id + `; + case "projection_thread_activities": + return sql>` + SELECT * + FROM projection_thread_activities + WHERE thread_id = ${threadId} + ORDER BY sequence, created_at, activity_id + `; + case "projection_turns": + return sql>` + SELECT + thread_id, + turn_id, + pending_message_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_turn_count, + checkpoint_ref, + checkpoint_status, + checkpoint_files_json, + source_proposed_plan_thread_id, + source_proposed_plan_id + FROM projection_turns + WHERE thread_id = ${threadId} + ORDER BY requested_at, row_id + `; + case "projection_thread_sessions": + return sql>` + SELECT * + FROM projection_thread_sessions + WHERE thread_id = ${threadId} + ORDER BY thread_id + `; + case "projection_thread_proposed_plans": + return sql>` + SELECT * + FROM projection_thread_proposed_plans + WHERE thread_id = ${threadId} + ORDER BY created_at, plan_id + `; + case "projection_pending_approvals": + return sql>` + SELECT * + FROM projection_pending_approvals + WHERE thread_id = ${threadId} + ORDER BY created_at, request_id + `; + } + })(); + return rows.pipe( + Effect.mapError((cause) => + cause instanceof Error + ? cause + : new Error(`Failed to read target checksum rows: ${String(cause)}`), + ), + ); +}; + +const checksumTables = [ + "projection_threads", + "projection_thread_messages", + "projection_thread_activities", + "projection_turns", + "projection_thread_sessions", + "projection_thread_proposed_plans", + "projection_pending_approvals", +] as const; + +function verifyTablesForMode( + mode: CliOptions["verify"], +): ReadonlyArray<(typeof checksumTables)[number]> { + switch (mode) { + case "none": + return []; + case "messages": + return ["projection_thread_messages"]; + case "full": + return checksumTables; + } +} + +function eventTimingBucket(event: OrchestrationEventType): string { + if (classifyReplayEvent(event) === "assistant-streaming-message") { + return "thread.message-sent:assistant-streaming"; + } + return event.type; +} + +function makeTargetLayer(options: CliOptions, targetFile: string | null) { + const persistence = + options.target === "memory" + ? SqlitePersistenceMemory + : makeSqlitePersistenceLive(targetFile ?? makeDefaultTargetFile(options.threadId)); + const repositoryIdentityResolverLayer = Layer.succeed(RepositoryIdentityResolver, { + resolve: () => Effect.succeed(null), + }); + + return Layer.mergeAll( + OrchestrationProjectionPipelineLive.pipe(Layer.provideMerge(OrchestrationEventStoreLive)), + OrchestrationProjectionSnapshotQueryLive.pipe(Layer.provide(repositoryIdentityResolverLayer)), + ).pipe( + Layer.provideMerge(persistence), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-benchmark-" })), + Layer.provideMerge(NodeServices.layer), + ); +} + +function makeDefaultTargetFile(threadId: string): string { + return join(tmpdir(), `t3-thread-replay-${threadId}-${Date.now()}.sqlite`); +} + +function formatMs(value: number): string { + return value.toFixed(4); +} + +function printStats(label: string, timings: ReadonlyArray) { + const stats = calculateTimingStats(timings); + console.log( + `${label}: count=${stats.count} total=${formatMs(stats.totalMs)}ms mean=${formatMs( + stats.meanMs, + )}ms p50=${formatMs(stats.p50Ms)}ms p90=${formatMs(stats.p90Ms)}ms p99=${formatMs( + stats.p99Ms, + )}ms max=${formatMs(stats.maxMs)}ms`, + ); +} + +function divideForSpeedup(left: number, right: number): number { + return right === 0 ? 0 : left / right; +} + +function speedupStats( + legacy: ReturnType, + optimized: ReturnType, +) { + return { + mean: divideForSpeedup(legacy.meanMs, optimized.meanMs), + p50: divideForSpeedup(legacy.p50Ms, optimized.p50Ms), + p90: divideForSpeedup(legacy.p90Ms, optimized.p90Ms), + p99: divideForSpeedup(legacy.p99Ms, optimized.p99Ms), + }; +} + +function printCompareStats(input: { + readonly label: string; + readonly optimized: ReadonlyArray; + readonly legacy: ReadonlyArray; +}) { + const optimized = calculateTimingStats(input.optimized); + const legacy = calculateTimingStats(input.legacy); + const speedup = speedupStats(legacy, optimized); + console.log(input.label); + console.log( + ` optimized: count=${optimized.count} mean=${formatMs(optimized.meanMs)}ms p50=${formatMs( + optimized.p50Ms, + )}ms p90=${formatMs(optimized.p90Ms)}ms p99=${formatMs(optimized.p99Ms)}ms max=${formatMs( + optimized.maxMs, + )}ms`, + ); + console.log( + ` legacy: count=${legacy.count} mean=${formatMs(legacy.meanMs)}ms p50=${formatMs( + legacy.p50Ms, + )}ms p90=${formatMs(legacy.p90Ms)}ms p99=${formatMs(legacy.p99Ms)}ms max=${formatMs( + legacy.maxMs, + )}ms`, + ); + console.log( + ` speedup: mean=${speedup.mean.toFixed(1)}x p50=${speedup.p50.toFixed( + 1, + )}x p90=${speedup.p90.toFixed(1)}x p99=${speedup.p99.toFixed(1)}x`, + ); +} + +function createAssistantStreamingCompareDb(input: { + readonly thread: Record; + readonly messages?: ReadonlyArray; + readonly activities?: ReadonlyArray>; +}) { + const db = new Database(":memory:", { strict: true }); + db.exec(` + PRAGMA journal_mode = MEMORY; + PRAGMA synchronous = OFF; + CREATE TABLE projection_threads ( + thread_id TEXT PRIMARY KEY, + project_id TEXT, + title TEXT, + branch TEXT, + worktree_path TEXT, + latest_turn_id TEXT, + created_at TEXT, + updated_at TEXT, + deleted_at TEXT, + runtime_mode TEXT, + interaction_mode TEXT, + model_selection_json TEXT, + archived_at TEXT, + latest_user_message_at TEXT, + pending_approval_count INTEGER DEFAULT 0, + pending_user_input_count INTEGER DEFAULT 0, + has_actionable_proposed_plan INTEGER DEFAULT 0 + ); + CREATE TABLE projection_thread_messages ( + message_id TEXT PRIMARY KEY, + thread_id TEXT, + turn_id TEXT, + role TEXT, + text TEXT, + is_streaming INTEGER, + created_at TEXT, + updated_at TEXT, + attachments_json TEXT + ); + CREATE INDEX idx_messages_thread_created + ON projection_thread_messages(thread_id, created_at, message_id); + CREATE TABLE projection_thread_activities ( + activity_id TEXT PRIMARY KEY, + thread_id TEXT, + turn_id TEXT, + tone TEXT, + kind TEXT, + summary TEXT, + payload_json TEXT, + created_at TEXT, + sequence INTEGER + ); + CREATE INDEX idx_activities_thread_sequence + ON projection_thread_activities(thread_id, sequence, created_at, activity_id); + CREATE TABLE projection_thread_proposed_plans ( + plan_id TEXT PRIMARY KEY, + thread_id TEXT, + turn_id TEXT, + plan_markdown TEXT, + created_at TEXT, + updated_at TEXT, + implemented_at TEXT, + implementation_thread_id TEXT + ); + CREATE TABLE projection_pending_approvals ( + request_id TEXT PRIMARY KEY, + thread_id TEXT, + turn_id TEXT, + status TEXT, + decision TEXT, + created_at TEXT, + resolved_at TEXT + ); + `); + + db.query(` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + branch, + worktree_path, + latest_turn_id, + created_at, + updated_at, + deleted_at, + runtime_mode, + interaction_mode, + model_selection_json, + archived_at, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan + ) + VALUES ( + $thread_id, + $project_id, + $title, + $branch, + $worktree_path, + $latest_turn_id, + $created_at, + $updated_at, + $deleted_at, + $runtime_mode, + $interaction_mode, + $model_selection_json, + $archived_at, + $latest_user_message_at, + $pending_approval_count, + $pending_user_input_count, + $has_actionable_proposed_plan + ) + `).run(input.thread as SqlBindingRecord); + + if (input.messages !== undefined || input.activities !== undefined) { + const insertMessage = db.query(` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + is_streaming, + created_at, + updated_at, + attachments_json + ) + VALUES ( + $messageId, + $threadId, + $turnId, + $role, + $text, + $isStreaming, + $createdAt, + $updatedAt, + $attachmentsJson + ) + `); + const insertActivity = db.query(` + INSERT INTO projection_thread_activities ( + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + created_at, + sequence + ) + VALUES ( + $activity_id, + $thread_id, + $turn_id, + $tone, + $kind, + $summary, + $payload_json, + $created_at, + $sequence + ) + `); + db.transaction(() => { + for (const message of input.messages ?? []) { + insertMessage.run(message as unknown as SqlBindingRecord); + } + for (const activity of input.activities ?? []) { + insertActivity.run(activity as SqlBindingRecord); + } + })(); + } + + return db; +} + +function selectSampledWindowEvents( + events: ReadonlyArray, + sampleSize: number, +): ReadonlyArray { + if (events.length <= sampleSize) { + return events; + } + const step = events.length / sampleSize; + const selected: Array = []; + for (let index = 0; index < sampleSize; index += 1) { + selected.push(events[Math.floor(index * step)]!); + } + return selected; +} + +function runAssistantStreamingCompare(options: CliOptions) { + const events = readAssistantStreamingEvents(options.sourceDb, options.threadId).slice( + 0, + options.limit ?? undefined, + ); + const thread = readSourceThreadRow(options.sourceDb, options.threadId); + const sourceMessages = readSourceMessageRows(options.sourceDb, options.threadId); + const sourceActivities = readSourceActivityRows(options.sourceDb, options.threadId); + const optimizedDb = createAssistantStreamingCompareDb({ thread }); + const optimizedAppend = optimizedDb.query(` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + is_streaming, + created_at, + updated_at, + attachments_json + ) + VALUES ( + $messageId, + $threadId, + $turnId, + $role, + $text, + 1, + $createdAt, + $updatedAt, + NULL + ) + ON CONFLICT(message_id) DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + role = excluded.role, + text = projection_thread_messages.text || excluded.text, + is_streaming = excluded.is_streaming, + updated_at = excluded.updated_at + `); + + const legacyDb = createAssistantStreamingCompareDb({ + thread, + messages: sourceMessages, + activities: sourceActivities, + }); + const legacyGet = legacyDb.query< + { readonly text: string; readonly createdAt: string; readonly attachmentsJson: string | null }, + [string] + >(` + SELECT + text, + created_at AS createdAt, + attachments_json AS attachmentsJson + FROM projection_thread_messages + WHERE message_id = ? + `); + const legacyUpsert = legacyDb.query(` + INSERT INTO projection_thread_messages ( + message_id, + thread_id, + turn_id, + role, + text, + is_streaming, + created_at, + updated_at, + attachments_json + ) + VALUES ( + $messageId, + $threadId, + $turnId, + $role, + $nextText, + 1, + $createdAtForWrite, + $updatedAt, + $attachmentsJson + ) + ON CONFLICT(message_id) DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + role = excluded.role, + text = excluded.text, + is_streaming = excluded.is_streaming, + updated_at = excluded.updated_at, + attachments_json = COALESCE( + excluded.attachments_json, + projection_thread_messages.attachments_json + ) + `); + const updateThread = legacyDb.query(` + UPDATE projection_threads + SET updated_at = ? + WHERE thread_id = ? + `); + const listMessages = legacyDb.query< + { readonly role: string; readonly createdAt: string }, + [string] + >(` + SELECT role, created_at AS createdAt + FROM projection_thread_messages + WHERE thread_id = ? + ORDER BY created_at ASC, message_id ASC + `); + const listActivities = legacyDb.query< + { readonly kind: string; readonly payloadJson: string; readonly createdAt: string }, + [string] + >(` + SELECT + kind, + payload_json AS payloadJson, + created_at AS createdAt + FROM projection_thread_activities + WHERE thread_id = ? + ORDER BY + CASE WHEN sequence IS NULL THEN 0 ELSE 1 END, + sequence ASC, + created_at ASC, + activity_id ASC + `); + const countApprovals = legacyDb.query<{ readonly count: number }, [string]>(` + SELECT count(*) AS count + FROM projection_pending_approvals + WHERE thread_id = ? + AND status = 'pending' + `); + const listPlans = legacyDb.query< + { + readonly planId: string; + readonly turnId: string | null; + readonly implementedAt: string | null; + }, + [string] + >(` + SELECT + plan_id AS planId, + turn_id AS turnId, + implemented_at AS implementedAt + FROM projection_thread_proposed_plans + WHERE thread_id = ? + ORDER BY updated_at ASC, plan_id ASC + `); + + const optimizedAll: Array = []; + const legacyAll: Array = []; + const windows: Array<{ + readonly label: string; + readonly optimized: ReadonlyArray; + readonly legacy: ReadonlyArray; + readonly optimizedCount: number; + readonly legacyCount: number; + }> = []; + + for (let start = 0; start < events.length; start += options.windowSize) { + const end = Math.min(events.length, start + options.windowSize); + const windowEvents = events.slice(start, end); + const optimizedWindow: Array = []; + for (const event of windowEvents) { + const startedAt = performance.now(); + optimizedAppend.run(event as unknown as SqlBindingRecord); + const elapsed = performance.now() - startedAt; + optimizedWindow.push(elapsed); + optimizedAll.push(elapsed); + } + + const legacyWindowEvents = + options.legacyMode === "full" + ? windowEvents + : selectSampledWindowEvents(windowEvents, options.legacySamplePerWindow); + const legacyWindow: Array = []; + for (const event of legacyWindowEvents) { + legacyDb.exec("BEGIN"); + const startedAt = performance.now(); + const existing = legacyGet.get(event.messageId); + const nextText = existing === null ? event.text : `${existing.text}${event.text}`; + legacyUpsert.run({ + ...event, + nextText, + attachmentsJson: existing?.attachmentsJson ?? null, + createdAtForWrite: existing?.createdAt ?? event.createdAt, + } as unknown as SqlBindingRecord); + updateThread.run(event.updatedAt, event.threadId); + const messages = listMessages.all(event.threadId); + const activities = listActivities.all(event.threadId); + const approvals = countApprovals.get(event.threadId)?.count ?? 0; + const plans = listPlans.all(event.threadId); + let latestUserMessageAt: string | null = null; + for (const message of messages) { + if (message.role === "user") { + latestUserMessageAt = message.createdAt; + } + } + let pendingUserInputCount = 0; + for (const activity of activities) { + if (activity.kind === "user-input.requested") { + pendingUserInputCount += 1; + } + } + void approvals; + void plans; + void latestUserMessageAt; + void pendingUserInputCount; + const elapsed = performance.now() - startedAt; + legacyWindow.push(elapsed); + legacyAll.push(elapsed); + legacyDb.exec("ROLLBACK"); + } + + windows.push({ + label: `window ${start + 1}-${end}`, + optimized: optimizedWindow, + legacy: legacyWindow, + optimizedCount: windowEvents.length, + legacyCount: legacyWindowEvents.length, + }); + console.log( + `compare progress ${end}/${events.length}: optimized=${windowEvents.length} legacy=${legacyWindowEvents.length}`, + ); + } + + console.log(`thread=${options.threadId}`); + console.log(`sourceDb=${options.sourceDb}`); + console.log("mode=compare assistant-streaming"); + console.log(`legacyMode=${options.legacyMode}`); + console.log(`windowSize=${options.windowSize}`); + console.log(`assistantStreamingEvents=${events.length}`); + console.log(`legacySamplePerWindow=${options.legacySamplePerWindow}`); + printCompareStats({ + label: "overall", + optimized: optimizedAll, + legacy: legacyAll, + }); + for (const window of windows) { + printCompareStats({ + label: `${window.label} optimizedEvents=${window.optimizedCount} legacyEvents=${window.legacyCount}`, + optimized: window.optimized, + legacy: window.legacy, + }); + } + + optimizedDb.close(); + legacyDb.close(); +} + +const runReplay = (options: CliOptions, targetFile: string | null) => + Effect.gen(function* () { + const sourceEvents = readSourceEvents(options.sourceDb, options.threadId).slice( + 0, + options.limit ?? undefined, + ); + const sourceProject = readSourceProject(options.sourceDb, options.threadId); + if (sourceEvents.length === 0) { + throw new Error(`No thread events found for ${options.threadId}`); + } + + const sql = yield* SqlClient.SqlClient; + if (sourceProject !== null) { + yield* sql` + INSERT INTO projection_projects ( + project_id, + title, + workspace_root, + scripts_json, + created_at, + updated_at, + deleted_at, + default_model_selection_json + ) + VALUES ( + ${sourceProject.projectId}, + ${sourceProject.title}, + ${sourceProject.workspaceRoot}, + ${sourceProject.scriptsJson}, + ${sourceProject.createdAt}, + ${sourceProject.updatedAt}, + ${sourceProject.deletedAt}, + ${sourceProject.defaultModelSelectionJson} + ) + `; + } + + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; + const allTimings: Array = []; + const assistantStreamingTimings: Array = []; + const otherTimings: Array = []; + const timingsByBucket = new Map>(); + const startedAt = performance.now(); + + for (const [index, event] of sourceEvents.entries()) { + const eventStartedAt = performance.now(); + yield* projectionPipeline.projectEvent(event); + const elapsedMs = performance.now() - eventStartedAt; + allTimings.push(elapsedMs); + const bucket = eventTimingBucket(event); + const bucketTimings = timingsByBucket.get(bucket); + if (bucketTimings === undefined) { + timingsByBucket.set(bucket, [elapsedMs]); + } else { + bucketTimings.push(elapsedMs); + } + if (classifyReplayEvent(event) === "assistant-streaming-message") { + assistantStreamingTimings.push(elapsedMs); + } else { + otherTimings.push(elapsedMs); + } + const replayed = index + 1; + if (replayed % options.progressEvery === 0 || replayed === sourceEvents.length) { + const stats = calculateTimingStats(allTimings.slice(-options.progressEvery)); + console.log( + `progress ${replayed}/${sourceEvents.length}: recent_mean=${formatMs( + stats.meanMs, + )}ms recent_p50=${formatMs(stats.p50Ms)}ms recent_p90=${formatMs( + stats.p90Ms, + )}ms recent_p99=${formatMs(stats.p99Ms)}ms`, + ); + } + } + + const totalWallMs = performance.now() - startedAt; + const detail = yield* projectionSnapshotQuery.getThreadDetailById( + ThreadId.make(options.threadId), + ); + if (Option.isNone(detail)) { + throw new Error("Replay completed but thread detail was not projected"); + } + + const targetChecksums: Record = {}; + const sourceChecksums: Record = {}; + const tablesToVerify = options.limit === null ? verifyTablesForMode(options.verify) : []; + if (tablesToVerify.length > 0) { + for (const table of tablesToVerify) { + const rows = yield* readTargetChecksumRows(sql, table, options.threadId); + targetChecksums[table] = checksumRows(rows); + } + + for (const table of tablesToVerify) { + sourceChecksums[table] = checksumRows( + readSourceChecksumRows(options.sourceDb, table, options.threadId), + ); + } + } + + return { + sourceEvents, + allTimings, + assistantStreamingTimings, + otherTimings, + timingsByBucket, + totalWallMs, + sourceChecksums, + targetChecksums, + detail: detail.value, + }; + }).pipe(Effect.provide(makeTargetLayer(options, targetFile))); + +function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.compare === "assistant-streaming") { + runAssistantStreamingCompare(options); + return; + } + + if (options.diffOnly) { + const sourceEvents = readSourceEvents(options.sourceDb, options.threadId).slice( + 0, + options.limit ?? undefined, + ); + const sourceRows = readSourceMessageRows(options.sourceDb, options.threadId); + const expectedRows = deriveMessageRowsFromEvents(sourceEvents); + const diffs = buildMessageDiffs({ + sourceRows, + expectedRows, + limit: options.diffMessages > 0 ? options.diffMessages : 20, + }); + console.log(`thread=${options.threadId}`); + console.log(`sourceDb=${options.sourceDb}`); + console.log(`mode=diff-only`); + console.log(`events=${sourceEvents.length}`); + console.log(`sourceMessages=${sourceRows.length}`); + console.log(`eventDerivedMessages=${expectedRows.length}`); + console.log(`message_diffs=${diffs.length}`); + for (const diff of diffs) { + console.log(` ${JSON.stringify(diff)}`); + } + return; + } + + const targetFile = + options.target === "file" + ? resolve(options.targetFile ?? makeDefaultTargetFile(options.threadId)) + : null; + if (targetFile !== null && existsSync(targetFile)) { + rmSync(targetFile, { force: true }); + } + + const result = Effect.runPromise( + runReplay(options, targetFile).pipe( + Effect.mapError((cause) => + cause instanceof Error ? cause : new Error(`Replay benchmark failed: ${String(cause)}`), + ), + ), + ); + result + .then((replay) => { + console.log(`thread=${options.threadId}`); + console.log(`sourceDb=${options.sourceDb}`); + console.log(`target=${options.target}${targetFile ? `:${targetFile}` : ""}`); + console.log(`events=${replay.sourceEvents.length}`); + console.log(`messages=${replay.detail.messages.length}`); + console.log(`activities=${replay.detail.activities.length}`); + console.log(`wall=${formatMs(replay.totalWallMs)}ms`); + printStats("all", replay.allTimings); + printStats("assistant_streaming", replay.assistantStreamingTimings); + printStats("other", replay.otherTimings); + console.log("event_type_buckets:"); + const sortedBuckets = [...replay.timingsByBucket.entries()].toSorted( + ([, left], [, right]) => right.length - left.length, + ); + for (const [bucket, timings] of sortedBuckets) { + printStats(` ${bucket}`, timings); + } + + console.log("samples:"); + for (const sample of buildTimingSamples(replay.allTimings, options.sampleEvery)) { + console.log( + ` ${sample.fromEvent}-${sample.toEvent}: mean=${formatMs( + sample.stats.meanMs, + )}ms p50=${formatMs(sample.stats.p50Ms)}ms p90=${formatMs( + sample.stats.p90Ms, + )}ms p99=${formatMs(sample.stats.p99Ms)}ms max=${formatMs(sample.stats.maxMs)}ms`, + ); + } + + const tablesToVerify = options.limit === null ? verifyTablesForMode(options.verify) : []; + if (tablesToVerify.length > 0) { + console.log(`checksums (${options.verify}):`); + let mismatchCount = 0; + for (const table of tablesToVerify) { + const source = replay.sourceChecksums[table]; + const target = replay.targetChecksums[table]; + const ok = source === target; + if (!ok) { + mismatchCount += 1; + } + console.log(` ${table}: ${ok ? "ok" : "mismatch"}`); + } + if (mismatchCount > 0) { + process.exitCode = 1; + } + } else if (options.verify === "none") { + console.log("checksums: skipped by --verify none"); + } else { + console.log("checksums: skipped for limited replay"); + } + if (options.diffMessages > 0) { + const sourceRows = readSourceMessageRows(options.sourceDb, options.threadId); + const expectedRows = deriveMessageRowsFromEvents(replay.sourceEvents); + const diffs = buildMessageDiffs({ + sourceRows, + expectedRows, + limit: options.diffMessages, + }); + console.log(`message_diffs: ${diffs.length}`); + for (const diff of diffs) { + console.log(` ${JSON.stringify(diff)}`); + } + } + if (targetFile !== null && !options.keepTarget) { + rmSync(targetFile, { force: true }); + rmSync(`${targetFile}-shm`, { force: true }); + rmSync(`${targetFile}-wal`, { force: true }); + } + }) + .catch((cause: unknown) => { + console.error(cause); + process.exitCode = 1; + }); +} + +if (import.meta.main) { + main(); +} diff --git a/apps/server/scripts/replay_thread/core.test.ts b/apps/server/scripts/replay_thread/core.test.ts new file mode 100644 index 0000000000..56c940d209 --- /dev/null +++ b/apps/server/scripts/replay_thread/core.test.ts @@ -0,0 +1,77 @@ +import { assert, it } from "@effect/vitest"; + +import { + buildTimingSamples, + calculateTimingStats, + checksumRows, + classifyReplayEvent, + stableJson, +} from "./core.ts"; + +it("calculates timing stats with stable percentile boundaries", () => { + assert.deepEqual(calculateTimingStats([]), { + count: 0, + totalMs: 0, + meanMs: 0, + p50Ms: 0, + p90Ms: 0, + p99Ms: 0, + maxMs: 0, + }); + + assert.deepEqual(calculateTimingStats([4, 1, 2, 3]), { + count: 4, + totalMs: 10, + meanMs: 2.5, + p50Ms: 2, + p90Ms: 4, + p99Ms: 4, + maxMs: 4, + }); +}); + +it("builds contiguous timing samples", () => { + assert.deepEqual(buildTimingSamples([1, 2, 3, 4, 5], 2), [ + { + fromEvent: 1, + toEvent: 2, + stats: calculateTimingStats([1, 2]), + }, + { + fromEvent: 3, + toEvent: 4, + stats: calculateTimingStats([3, 4]), + }, + { + fromEvent: 5, + toEvent: 5, + stats: calculateTimingStats([5]), + }, + ]); +}); + +it("hashes rows independently of object key insertion order", () => { + const left = [{ b: 2, a: { d: 4, c: 3 } }]; + const right = [{ a: { c: 3, d: 4 }, b: 2 }]; + + assert.equal(stableJson(left), stableJson(right)); + assert.equal(checksumRows(left), checksumRows(right)); +}); + +it("classifies assistant streaming events separately from other events", () => { + assert.equal( + classifyReplayEvent({ + type: "thread.message-sent", + payload: { role: "assistant", streaming: true }, + }), + "assistant-streaming-message", + ); + assert.equal( + classifyReplayEvent({ + type: "thread.message-sent", + payload: { role: "assistant", streaming: false }, + }), + "other", + ); + assert.equal(classifyReplayEvent({ type: "thread.created", payload: {} }), "other"); +}); diff --git a/apps/server/scripts/replay_thread/core.ts b/apps/server/scripts/replay_thread/core.ts new file mode 100644 index 0000000000..e714e64e80 --- /dev/null +++ b/apps/server/scripts/replay_thread/core.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; + +export interface TimingStats { + readonly count: number; + readonly totalMs: number; + readonly meanMs: number; + readonly p50Ms: number; + readonly p90Ms: number; + readonly p99Ms: number; + readonly maxMs: number; +} + +export interface ReplayTimingSample { + readonly fromEvent: number; + readonly toEvent: number; + readonly stats: TimingStats; +} + +export function calculateTimingStats(values: ReadonlyArray): TimingStats { + if (values.length === 0) { + return { + count: 0, + totalMs: 0, + meanMs: 0, + p50Ms: 0, + p90Ms: 0, + p99Ms: 0, + maxMs: 0, + }; + } + + const sorted = [...values].toSorted((left, right) => left - right); + const totalMs = values.reduce((total, value) => total + value, 0); + const percentile = (percent: number) => { + const index = Math.min(sorted.length - 1, Math.ceil((percent / 100) * sorted.length) - 1); + return sorted[Math.max(0, index)] ?? 0; + }; + + return { + count: values.length, + totalMs, + meanMs: totalMs / values.length, + p50Ms: percentile(50), + p90Ms: percentile(90), + p99Ms: percentile(99), + maxMs: sorted[sorted.length - 1] ?? 0, + }; +} + +export function buildTimingSamples( + timings: ReadonlyArray, + sampleEvery: number, +): ReadonlyArray { + const normalizedSampleEvery = Math.max(1, Math.floor(sampleEvery)); + const samples: Array = []; + for (let start = 0; start < timings.length; start += normalizedSampleEvery) { + const end = Math.min(timings.length, start + normalizedSampleEvery); + samples.push({ + fromEvent: start + 1, + toEvent: end, + stats: calculateTimingStats(timings.slice(start, end)), + }); + } + return samples; +} + +function stableValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(stableValue); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, stableValue(nested)]), + ); + } + return value; +} + +export function stableJson(value: unknown): string { + return JSON.stringify(stableValue(value)); +} + +export function checksumRows(rows: ReadonlyArray>): string { + const hash = createHash("sha256"); + for (const row of rows) { + hash.update(stableJson(row)); + hash.update("\n"); + } + return hash.digest("hex"); +} + +export function classifyReplayEvent(event: { + readonly type: string; + readonly payload: unknown; +}): "assistant-streaming-message" | "other" { + if (event.type !== "thread.message-sent") { + return "other"; + } + const payload = + typeof event.payload === "object" && event.payload !== null + ? (event.payload as Record) + : {}; + return payload.role === "assistant" && payload.streaming === true + ? "assistant-streaming-message" + : "other"; +} From e8264648bcf6ad59cb2dcfb07e37b5ea9c503572 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Sun, 10 May 2026 13:39:55 +0100 Subject: [PATCH 04/10] Cover streaming append snapshot output --- .../Layers/ProjectionPipeline.test.ts | 366 +++++++++++++----- 1 file changed, 261 insertions(+), 105 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index df1fd6bc9a..94028bf2b3 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -33,6 +33,7 @@ import { import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { OrchestrationProjectionPipeline } from "../Services/ProjectionPipeline.ts"; +import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { ServerConfig } from "../../config.ts"; const makeProjectionPipelinePrefixedTestLayer = (prefix: string) => @@ -596,106 +597,113 @@ it.layer( ); }); -it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-streaming-append-")))( - "OrchestrationProjectionPipeline", - (it) => { - it.effect("appends assistant streaming deltas in-place while preserving row metadata", () => - Effect.gen(function* () { - const projectionPipeline = yield* OrchestrationProjectionPipeline; - const eventStore = yield* OrchestrationEventStore; - const sql = yield* SqlClient.SqlClient; - const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); - const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); - const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); +it.layer( + Layer.fresh( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provideMerge( + makeProjectionPipelinePrefixedTestLayer("t3-projection-streaming-append-"), + ), + Layer.provideMerge(RepositoryIdentityResolverLive), + ), + ), +)("OrchestrationProjectionPipeline", (it) => { + it.effect("appends assistant streaming deltas in-place while preserving row metadata", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); + const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); + const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); - const appendAndProject = (event: Parameters[0]) => - eventStore - .append(event) - .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent))); + const appendAndProject = (event: Parameters[0]) => + eventStore + .append(event) + .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent))); - yield* appendAndProject({ - type: "thread.message-sent", - eventId: EventId.make("evt-streaming-append-1"), - aggregateKind: "thread", - aggregateId: ThreadId.make("thread-streaming-append"), - occurredAt: first, - commandId: CommandId.make("cmd-streaming-append-1"), - causationEventId: null, - correlationId: CommandId.make("cmd-streaming-append-1"), - metadata: {}, - payload: { - threadId: ThreadId.make("thread-streaming-append"), - messageId: MessageId.make("message-streaming-append"), - role: "assistant", - text: "Hello", - attachments: [ - { - type: "image", - id: "thread-streaming-append-att-1", - name: "first.png", - mimeType: "image/png", - sizeBytes: 5, - }, - ], - turnId: TurnId.make("turn-streaming-append"), - streaming: true, - createdAt: first, - updatedAt: first, - }, - }); + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-1"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: first, + commandId: CommandId.make("cmd-streaming-append-1"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-1"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: "Hello", + attachments: [ + { + type: "image", + id: "thread-streaming-append-att-1", + name: "first.png", + mimeType: "image/png", + sizeBytes: 5, + }, + ], + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: first, + updatedAt: first, + }, + }); - yield* appendAndProject({ - type: "thread.message-sent", - eventId: EventId.make("evt-streaming-append-2"), - aggregateKind: "thread", - aggregateId: ThreadId.make("thread-streaming-append"), - occurredAt: second, - commandId: CommandId.make("cmd-streaming-append-2"), - causationEventId: null, - correlationId: CommandId.make("cmd-streaming-append-2"), - metadata: {}, - payload: { - threadId: ThreadId.make("thread-streaming-append"), - messageId: MessageId.make("message-streaming-append"), - role: "assistant", - text: " world", - turnId: TurnId.make("turn-streaming-append"), - streaming: true, - createdAt: second, - updatedAt: second, - }, - }); + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-2"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: second, + commandId: CommandId.make("cmd-streaming-append-2"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-2"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: " world", + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: second, + updatedAt: second, + }, + }); - yield* appendAndProject({ - type: "thread.message-sent", - eventId: EventId.make("evt-streaming-append-3"), - aggregateKind: "thread", - aggregateId: ThreadId.make("thread-streaming-append"), - occurredAt: third, - commandId: CommandId.make("cmd-streaming-append-3"), - causationEventId: null, - correlationId: CommandId.make("cmd-streaming-append-3"), - metadata: {}, - payload: { - threadId: ThreadId.make("thread-streaming-append"), - messageId: MessageId.make("message-streaming-append"), - role: "assistant", - text: "!", - attachments: [], - turnId: TurnId.make("turn-streaming-append"), - streaming: true, - createdAt: third, - updatedAt: third, - }, - }); + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-streaming-append-3"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-streaming-append"), + occurredAt: third, + commandId: CommandId.make("cmd-streaming-append-3"), + causationEventId: null, + correlationId: CommandId.make("cmd-streaming-append-3"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-streaming-append"), + messageId: MessageId.make("message-streaming-append"), + role: "assistant", + text: "!", + attachments: [], + turnId: TurnId.make("turn-streaming-append"), + streaming: true, + createdAt: third, + updatedAt: third, + }, + }); - const rows = yield* sql<{ - readonly text: string; - readonly attachmentsJson: string | null; - readonly createdAt: string; - readonly updatedAt: string; - readonly isStreaming: number; - }>` + const rows = yield* sql<{ + readonly text: string; + readonly attachmentsJson: string | null; + readonly createdAt: string; + readonly updatedAt: string; + readonly isStreaming: number; + }>` SELECT text, attachments_json AS "attachmentsJson", @@ -706,18 +714,166 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-stre WHERE message_id = 'message-streaming-append' `; - assert.equal(rows.length, 1); - assert.deepEqual(rows[0], { - text: "Hello world!", - attachmentsJson: "[]", + assert.equal(rows.length, 1); + assert.deepEqual(rows[0], { + text: "Hello world!", + attachmentsJson: "[]", + createdAt: first, + updatedAt: third, + isStreaming: 1, + }); + }), + ); + + it.effect("exposes appended assistant streaming text through thread detail snapshots", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const snapshotQuery = yield* ProjectionSnapshotQuery; + const eventStore = yield* OrchestrationEventStore; + const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); + const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); + const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); + + const appendAndProject = (event: Parameters[0]) => + eventStore + .append(event) + .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent))); + + yield* appendAndProject({ + type: "project.created", + eventId: EventId.make("evt-client-visible-1"), + aggregateKind: "project", + aggregateId: ProjectId.make("project-client-visible"), + occurredAt: first, + commandId: CommandId.make("cmd-client-visible-1"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-client-visible-1"), + metadata: {}, + payload: { + projectId: ProjectId.make("project-client-visible"), + title: "Project", + workspaceRoot: "/tmp/project-client-visible", + defaultModelSelection: null, + scripts: [], + createdAt: first, + updatedAt: first, + }, + }); + + yield* appendAndProject({ + type: "thread.created", + eventId: EventId.make("evt-client-visible-2"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-client-visible"), + occurredAt: first, + commandId: CommandId.make("cmd-client-visible-2"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-client-visible-2"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-client-visible"), + projectId: ProjectId.make("project-client-visible"), + title: "Thread", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdAt: first, + updatedAt: first, + }, + }); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-client-visible-3"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-client-visible"), + occurredAt: first, + commandId: CommandId.make("cmd-client-visible-3"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-client-visible-3"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-client-visible"), + messageId: MessageId.make("message-client-visible"), + role: "assistant", + text: "Hello", + turnId: null, + streaming: true, createdAt: first, + updatedAt: first, + }, + }); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-client-visible-4"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-client-visible"), + occurredAt: second, + commandId: CommandId.make("cmd-client-visible-4"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-client-visible-4"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-client-visible"), + messageId: MessageId.make("message-client-visible"), + role: "assistant", + text: " world", + turnId: null, + streaming: true, + createdAt: second, + updatedAt: second, + }, + }); + + yield* appendAndProject({ + type: "thread.message-sent", + eventId: EventId.make("evt-client-visible-5"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-client-visible"), + occurredAt: third, + commandId: CommandId.make("cmd-client-visible-5"), + causationEventId: null, + correlationId: CorrelationId.make("cmd-client-visible-5"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-client-visible"), + messageId: MessageId.make("message-client-visible"), + role: "assistant", + text: "", + turnId: null, + streaming: false, + createdAt: third, updatedAt: third, - isStreaming: 1, - }); - }), - ); - }, -); + }, + }); + + const detail = yield* snapshotQuery.getThreadDetailById( + ThreadId.make("thread-client-visible"), + ); + + assert.equal(detail._tag, "Some"); + if (detail._tag === "Some") { + assert.deepEqual(detail.value.messages, [ + { + id: MessageId.make("message-client-visible"), + role: "assistant", + text: "Hello world", + turnId: null, + streaming: false, + createdAt: first, + updatedAt: third, + }, + ]); + } + }), + ); +}); it.layer( Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-attachments-rollback-")), From 102f5a93406e258b7920259294c04dd2fee5383a Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Sun, 10 May 2026 13:54:56 +0100 Subject: [PATCH 05/10] Slim assistant streaming benchmark --- .../server/scripts/replay_thread/benchmark.ts | 1852 +++++------------ .../server/scripts/replay_thread/core.test.ts | 34 +- apps/server/scripts/replay_thread/core.ts | 45 - 3 files changed, 541 insertions(+), 1390 deletions(-) diff --git a/apps/server/scripts/replay_thread/benchmark.ts b/apps/server/scripts/replay_thread/benchmark.ts index c58c8e8454..70d85e885f 100644 --- a/apps/server/scripts/replay_thread/benchmark.ts +++ b/apps/server/scripts/replay_thread/benchmark.ts @@ -1,77 +1,53 @@ -import { existsSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { existsSync } from "node:fs"; import { performance } from "node:perf_hooks"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { - OrchestrationEvent, - ThreadId, - type OrchestrationEvent as OrchestrationEventType, -} from "@t3tools/contracts"; import { Database } from "bun:sqlite"; -import { Effect, Layer, Option, Schema } from "effect"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; -import { ServerConfig } from "../../src/config.ts"; -import { OrchestrationEventStoreLive } from "../../src/persistence/Layers/OrchestrationEventStore.ts"; -import { - makeSqlitePersistenceLive, - SqlitePersistenceMemory, -} from "../../src/persistence/Layers/Sqlite.ts"; -import { RepositoryIdentityResolver } from "../../src/project/Services/RepositoryIdentityResolver.ts"; -import { OrchestrationProjectionPipelineLive } from "../../src/orchestration/Layers/ProjectionPipeline.ts"; -import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; -import { OrchestrationProjectionPipeline } from "../../src/orchestration/Services/ProjectionPipeline.ts"; -import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; -import { - buildTimingSamples, - calculateTimingStats, - checksumRows, - classifyReplayEvent, -} from "./core.ts"; +import { calculateTimingStats } from "./core.ts"; interface CliOptions { readonly sourceDb: string; readonly threadId: string; - readonly target: "memory" | "file"; - readonly targetFile: string | null; - readonly verify: "none" | "messages" | "full"; - readonly diffMessages: number; - readonly diffOnly: boolean; - readonly compare: "none" | "assistant-streaming"; + readonly windowSize: number; readonly legacyMode: "sampled" | "full"; readonly legacySamplePerWindow: number; - readonly windowSize: number; - readonly sampleEvery: number; - readonly progressEvery: number; readonly limit: number | null; - readonly keepTarget: boolean; } interface SourceEventRow { readonly sequence: number; - readonly eventId: string; - readonly aggregateKind: "project" | "thread"; - readonly aggregateId: string; - readonly type: string; - readonly occurredAt: string; - readonly commandId: string | null; - readonly causationEventId: string | null; - readonly correlationId: string | null; - readonly payload: string; - readonly metadata: string; + readonly payloadJson: string; +} + +interface AssistantStreamingEvent { + readonly sequence: number; + readonly messageId: string; + readonly threadId: string; + readonly turnId: string | null; + readonly role: "assistant"; + readonly text: string; + readonly createdAt: string; + readonly updatedAt: string; } -interface SourceProjectRow { +interface SourceThreadRow { + readonly threadId: string; readonly projectId: string; readonly title: string; - readonly workspaceRoot: string; - readonly scriptsJson: string; + readonly modelSelectionJson: string | null; + readonly runtimeMode: string; + readonly interactionMode: string; + readonly branch: string | null; + readonly worktreePath: string | null; + readonly latestTurnId: string | null; readonly createdAt: string; readonly updatedAt: string; + readonly archivedAt: string | null; + readonly latestUserMessageAt: string | null; + readonly pendingApprovalCount: number; + readonly pendingUserInputCount: number; + readonly hasActionableProposedPlan: number; readonly deletedAt: string | null; - readonly defaultModelSelectionJson: string | null; } interface SourceMessageRow { @@ -86,36 +62,49 @@ interface SourceMessageRow { readonly updatedAt: string; } -interface AssistantStreamingEventRow { - readonly messageId: string; +interface SourceActivityRow { + readonly activityId: string; readonly threadId: string; readonly turnId: string | null; - readonly role: string; - readonly text: string; + readonly tone: string; + readonly kind: string; + readonly summary: string; + readonly payloadJson: string; + readonly sequence: number | null; + readonly createdAt: string; +} + +interface SourcePlanRow { + readonly planId: string; + readonly threadId: string; + readonly turnId: string | null; + readonly planMarkdown: string; + readonly implementedAt: string | null; + readonly implementationThreadId: string | null; + readonly createdAt: string; + readonly updatedAt: string; +} + +interface SourceApprovalRow { + readonly approvalId: string; + readonly threadId: string; + readonly turnId: string | null; + readonly status: string; readonly createdAt: string; readonly updatedAt: string; } type SqlBindingRecord = Record; -const decodeEvent = Schema.decodeUnknownSync(OrchestrationEvent); +let _summaryRefreshSink = ""; function parseArgs(argv: ReadonlyArray): CliOptions { let sourceDb: string | undefined; let threadId: string | undefined; - let target: CliOptions["target"] = "memory"; - let targetFile: string | null = null; - let verify: CliOptions["verify"] = "messages"; - let diffMessages = 0; - let diffOnly = false; - let compare: CliOptions["compare"] = "none"; + let windowSize = 10_000; let legacyMode: CliOptions["legacyMode"] = "sampled"; let legacySamplePerWindow = 500; - let windowSize = 10_000; - let sampleEvery = 1_000; - let progressEvery = 10_000; let limit: number | null = null; - let keepTarget = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -135,40 +124,9 @@ function parseArgs(argv: ReadonlyArray): CliOptions { case "--thread-id": threadId = next(); break; - case "--target": { - const value = next(); - if (value !== "memory" && value !== "file") { - throw new Error("--target must be memory or file"); - } - target = value; - break; - } - case "--target-file": - targetFile = next(); - target = "file"; - break; - case "--verify": { - const value = next(); - if (value !== "none" && value !== "messages" && value !== "full") { - throw new Error("--verify must be none, messages, or full"); - } - verify = value; - break; - } - case "--diff-messages": - diffMessages = Number(next()); - break; - case "--diff-only": - diffOnly = true; - break; - case "--compare": { - const value = next(); - if (value !== "assistant-streaming") { - throw new Error("--compare must be assistant-streaming"); - } - compare = value; + case "--window-size": + windowSize = Number(next()); break; - } case "--legacy-mode": { const value = next(); if (value !== "sampled" && value !== "full") { @@ -180,21 +138,9 @@ function parseArgs(argv: ReadonlyArray): CliOptions { case "--legacy-sample-per-window": legacySamplePerWindow = Number(next()); break; - case "--window-size": - windowSize = Number(next()); - break; - case "--sample-every": - sampleEvery = Number(next()); - break; - case "--progress-every": - progressEvery = Number(next()); - break; case "--limit": limit = Number(next()); break; - case "--keep-target": - keepTarget = true; - break; case "--help": case "-h": printUsage(); @@ -210,41 +156,23 @@ function parseArgs(argv: ReadonlyArray): CliOptions { if (!threadId) { throw new Error("Missing --thread-id"); } - if (!Number.isFinite(sampleEvery) || sampleEvery < 1) { - throw new Error("--sample-every must be a positive number"); - } - if (!Number.isFinite(diffMessages) || diffMessages < 0) { - throw new Error("--diff-messages must be zero or a positive number"); - } - if (!Number.isFinite(progressEvery) || progressEvery < 1) { - throw new Error("--progress-every must be a positive number"); + if (!Number.isFinite(windowSize) || windowSize < 1) { + throw new Error("--window-size must be a positive number"); } if (!Number.isFinite(legacySamplePerWindow) || legacySamplePerWindow < 1) { throw new Error("--legacy-sample-per-window must be a positive number"); } - if (!Number.isFinite(windowSize) || windowSize < 1) { - throw new Error("--window-size must be a positive number"); - } if (limit !== null && (!Number.isFinite(limit) || limit < 1)) { throw new Error("--limit must be a positive number"); } return { - sourceDb: resolve(sourceDb), + sourceDb, threadId, - target, - targetFile, - verify, - diffMessages: Math.floor(diffMessages), - diffOnly, - compare, + windowSize, legacyMode, - legacySamplePerWindow: Math.floor(legacySamplePerWindow), - windowSize: Math.floor(windowSize), - sampleEvery: Math.floor(sampleEvery), - progressEvery: Math.floor(progressEvery), - limit: limit === null ? null : Math.floor(limit), - keepTarget, + legacySamplePerWindow, + limit, }; } @@ -252,870 +180,339 @@ function printUsage() { console.log(`Usage: bun apps/server/scripts/replay_thread/benchmark.ts \\ --source-db C:\\Users\\mike\\.t3\\dev\\state.sqlite \\ - --thread-id de1c398f-5d3c-40e4-911c-2b672653cda7 + --thread-id de1c398f-5d3c-40e4-911c-2b672653cda7 \\ + --window-size 10000 \\ + --legacy-sample-per-window 500 Options: - --target memory|file Replay into an isolated in-memory DB by default. - --target-file Replay into a temp/inspectable SQLite file. - --verify none|messages|full - Verify messages by default. Full checks every table. - --diff-messages Print the first n source-vs-event message diffs. - --diff-only Only run message diffing; skip projection replay. - --compare assistant-streaming - Compare optimized vs legacy assistant streaming paths. - --legacy-mode sampled|full - Default: sampled. Full can take hours on large threads. - --legacy-sample-per-window - Legacy samples per window in sampled mode. Default: 500. - --window-size Assistant-streaming compare window size. Default: 10000. - --sample-every Print timing windows of n events. Default: 1000. - --progress-every Print progress every n events. Default: 10000. - --limit Replay only the first n events for smoke testing. - --keep-target Keep target file when --target-file is used. + --source-db SQLite DB containing the copied production thread + --thread-id Thread to benchmark + --window-size Assistant-streaming events per output window (default: 10000) + --legacy-mode sampled|full Sample old path per window, or run every event (default: sampled) + --legacy-sample-per-window Old-path samples per window in sampled mode (default: 500) + --limit Optional cap for smoke runs `); } -function readSourceEvents( - sourceDb: string, - threadId: string, -): ReadonlyArray { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - const rows = db - .query(` - SELECT - sequence, - event_id AS eventId, - aggregate_kind AS aggregateKind, - stream_id AS aggregateId, - event_type AS type, - occurred_at AS occurredAt, - command_id AS commandId, - causation_event_id AS causationEventId, - correlation_id AS correlationId, - payload_json AS payload, - metadata_json AS metadata - FROM orchestration_events - WHERE aggregate_kind = 'thread' - AND stream_id = ? - ORDER BY sequence ASC - `) - .all(threadId); - return rows.map((row) => - decodeEvent({ - ...row, - payload: JSON.parse(row.payload), - metadata: JSON.parse(row.metadata), - }), - ); - } finally { - db.close(); - } -} - -function readSourceProject(sourceDb: string, threadId: string): SourceProjectRow | null { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - return ( - db - .query(` - SELECT - p.project_id AS projectId, - p.title, - p.workspace_root AS workspaceRoot, - p.scripts_json AS scriptsJson, - p.created_at AS createdAt, - p.updated_at AS updatedAt, - p.deleted_at AS deletedAt, - p.default_model_selection_json AS defaultModelSelectionJson - FROM projection_threads t - JOIN projection_projects p ON p.project_id = t.project_id - WHERE t.thread_id = ? - LIMIT 1 - `) - .get(threadId) ?? null - ); - } finally { - db.close(); +function openSourceDb(path: string): Database { + if (!existsSync(path)) { + throw new Error(`Source DB does not exist: ${path}`); } + return new Database(path, { readonly: true, strict: true }); } function readAssistantStreamingEvents( - sourceDb: string, - threadId: string, -): ReadonlyArray { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - return db - .query(` - SELECT - json_extract(payload_json, '$.messageId') AS messageId, - json_extract(payload_json, '$.threadId') AS threadId, - json_extract(payload_json, '$.turnId') AS turnId, - json_extract(payload_json, '$.role') AS role, - json_extract(payload_json, '$.text') AS text, - json_extract(payload_json, '$.createdAt') AS createdAt, - json_extract(payload_json, '$.updatedAt') AS updatedAt + db: Database, + options: CliOptions, +): ReadonlyArray { + const rows = db + .query( + ` + SELECT sequence, payload_json AS payloadJson FROM orchestration_events - WHERE aggregate_kind = 'thread' - AND stream_id = ? - AND event_type = 'thread.message-sent' - AND json_extract(payload_json, '$.role') = 'assistant' - AND json_extract(payload_json, '$.streaming') = 1 + WHERE stream_id = ? AND event_type = 'thread.message-sent' ORDER BY sequence ASC - `) - .all(threadId); - } finally { - db.close(); - } -} - -function readSourceThreadRow(sourceDb: string, threadId: string): Record { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - const row = db - .query, [string]>(` - SELECT - thread_id, - project_id, - title, - branch, - worktree_path, - latest_turn_id, - created_at, - updated_at, - deleted_at, - runtime_mode, - interaction_mode, - model_selection_json, - archived_at, - latest_user_message_at, - pending_approval_count, - pending_user_input_count, - has_actionable_proposed_plan - FROM projection_threads - WHERE thread_id = ? - `) - .get(threadId); - if (row === null) { - throw new Error(`Thread ${threadId} not found in source DB`); - } - return row; - } finally { - db.close(); - } -} - -function readSourceActivityRows( - sourceDb: string, - threadId: string, -): ReadonlyArray> { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - return db - .query, [string]>(` - SELECT - activity_id, - thread_id, - turn_id, - tone, - kind, - summary, - payload_json, - created_at, - sequence - FROM projection_thread_activities - WHERE thread_id = ? - `) - .all(threadId); - } finally { - db.close(); - } -} - -function readSourceChecksumRows( - sourceDb: string, - table: string, - threadId: string, -): ReadonlyArray> { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - return db.query, [string]>(checksumQuery(table)).all(threadId); - } finally { - db.close(); - } -} - -function readSourceMessageRows( - sourceDb: string, - threadId: string, -): ReadonlyArray { - const db = new Database(sourceDb, { readonly: true, strict: true }); - try { - return db - .query(` - SELECT - message_id AS messageId, - thread_id AS threadId, - turn_id AS turnId, - role, - text, - attachments_json AS attachmentsJson, - is_streaming AS isStreaming, - created_at AS createdAt, - updated_at AS updatedAt - FROM projection_thread_messages - WHERE thread_id = ? - ORDER BY created_at, message_id - `) - .all(threadId); - } finally { - db.close(); - } -} - -function stringifyAttachments(attachments: unknown): string | null { - return attachments === undefined ? null : JSON.stringify(attachments); -} + `, + ) + .all(options.threadId); -function deriveMessageRowsFromEvents( - events: ReadonlyArray, -): ReadonlyArray { - const messages = new Map(); - for (const event of events) { - if (event.type !== "thread.message-sent") { + const events: Array = []; + for (const row of rows) { + const payload = JSON.parse(row.payloadJson) as Partial & { + readonly streaming?: unknown; + }; + if ( + payload.threadId !== options.threadId || + payload.role !== "assistant" || + payload.streaming !== true || + typeof payload.messageId !== "string" || + typeof payload.text !== "string" || + typeof payload.createdAt !== "string" || + typeof payload.updatedAt !== "string" + ) { continue; } - const previous = messages.get(event.payload.messageId); - const text = - previous === undefined - ? event.payload.text - : event.payload.streaming - ? `${previous.text}${event.payload.text}` - : event.payload.text.length === 0 - ? previous.text - : event.payload.text; - const attachments = - event.payload.attachments !== undefined - ? event.payload.attachments - : previous?.attachmentsJson !== null && previous?.attachmentsJson !== undefined - ? JSON.parse(previous.attachmentsJson) - : undefined; - messages.set(event.payload.messageId, { - messageId: event.payload.messageId, - threadId: event.payload.threadId, - turnId: event.payload.turnId, - role: event.payload.role, - text, - attachmentsJson: stringifyAttachments(attachments), - isStreaming: event.payload.streaming ? 1 : 0, - createdAt: previous?.createdAt ?? event.payload.createdAt, - updatedAt: event.payload.updatedAt, + events.push({ + sequence: row.sequence, + messageId: payload.messageId, + threadId: options.threadId, + turnId: typeof payload.turnId === "string" ? payload.turnId : null, + role: "assistant", + text: payload.text, + createdAt: payload.createdAt, + updatedAt: payload.updatedAt, }); - } - return [...messages.values()].toSorted( - (left, right) => - left.createdAt.localeCompare(right.createdAt) || - left.messageId.localeCompare(right.messageId), - ); -} - -function compactValue(value: unknown): unknown { - if (typeof value === "string" && value.length > 180) { - return `${value.slice(0, 180)}...`; - } - return value; -} - -function buildMessageDiffs(input: { - readonly sourceRows: ReadonlyArray; - readonly expectedRows: ReadonlyArray; - readonly limit: number; -}): ReadonlyArray> { - if (input.limit <= 0) { - return []; - } - const sourceById = new Map(input.sourceRows.map((row) => [row.messageId, row] as const)); - const expectedById = new Map(input.expectedRows.map((row) => [row.messageId, row] as const)); - const diffs: Array> = []; - const keys = [ - "threadId", - "turnId", - "role", - "text", - "attachmentsJson", - "isStreaming", - "createdAt", - "updatedAt", - ] as const; - - for (const [messageId, expected] of expectedById) { - const source = sourceById.get(messageId); - if (source === undefined) { - diffs.push({ messageId, kind: "missing-source" }); - } else { - for (const key of keys) { - if (source[key] !== expected[key]) { - const doubled = - key === "text" && - typeof source.text === "string" && - source.text.length === expected.text.length * 2 && - source.text === `${expected.text}${expected.text}`; - diffs.push({ - messageId, - key, - source: compactValue(source[key]), - expected: compactValue(expected[key]), - sourceLength: typeof source[key] === "string" ? source[key].length : undefined, - expectedLength: typeof expected[key] === "string" ? expected[key].length : undefined, - ...(doubled ? { looksDoubled: true } : {}), - }); - break; - } - } - } - if (diffs.length >= input.limit) { - return diffs; - } - } - - for (const messageId of sourceById.keys()) { - if (!expectedById.has(messageId)) { - diffs.push({ messageId, kind: "extra-source" }); - } - if (diffs.length >= input.limit) { - return diffs; - } - } - - return diffs; -} - -function checksumQuery(table: string): string { - switch (table) { - case "projection_threads": - return ` - SELECT - thread_id, - project_id, - title, - branch, - worktree_path, - latest_turn_id, - created_at, - updated_at, - deleted_at, - runtime_mode, - interaction_mode, - model_selection_json, - archived_at, - latest_user_message_at, - pending_approval_count, - pending_user_input_count, - has_actionable_proposed_plan - FROM projection_threads - WHERE thread_id = ? - ORDER BY thread_id - `; - case "projection_thread_messages": - return ` - SELECT * - FROM projection_thread_messages - WHERE thread_id = ? - ORDER BY created_at, message_id - `; - case "projection_thread_activities": - return ` - SELECT * - FROM projection_thread_activities - WHERE thread_id = ? - ORDER BY sequence, created_at, activity_id - `; - case "projection_turns": - return ` - SELECT - thread_id, - turn_id, - pending_message_id, - assistant_message_id, - state, - requested_at, - started_at, - completed_at, - checkpoint_turn_count, - checkpoint_ref, - checkpoint_status, - checkpoint_files_json, - source_proposed_plan_thread_id, - source_proposed_plan_id - FROM projection_turns - WHERE thread_id = ? - ORDER BY requested_at, row_id - `; - case "projection_thread_sessions": - return ` - SELECT * - FROM projection_thread_sessions - WHERE thread_id = ? - ORDER BY thread_id - `; - case "projection_thread_proposed_plans": - return ` - SELECT * - FROM projection_thread_proposed_plans - WHERE thread_id = ? - ORDER BY created_at, plan_id - `; - case "projection_pending_approvals": - return ` - SELECT * - FROM projection_pending_approvals - WHERE thread_id = ? - ORDER BY created_at, request_id - `; - default: - throw new Error(`Unsupported checksum table: ${table}`); - } -} - -const readTargetChecksumRows = ( - sql: SqlClient.SqlClient, - table: (typeof checksumTables)[number], - threadId: string, -): Effect.Effect>, Error> => { - const rows = (() => { - switch (table) { - case "projection_threads": - return sql>` - SELECT - thread_id, - project_id, - title, - branch, - worktree_path, - latest_turn_id, - created_at, - updated_at, - deleted_at, - runtime_mode, - interaction_mode, - model_selection_json, - archived_at, - latest_user_message_at, - pending_approval_count, - pending_user_input_count, - has_actionable_proposed_plan - FROM projection_threads - WHERE thread_id = ${threadId} - ORDER BY thread_id - `; - case "projection_thread_messages": - return sql>` - SELECT * - FROM projection_thread_messages - WHERE thread_id = ${threadId} - ORDER BY created_at, message_id - `; - case "projection_thread_activities": - return sql>` - SELECT * - FROM projection_thread_activities - WHERE thread_id = ${threadId} - ORDER BY sequence, created_at, activity_id - `; - case "projection_turns": - return sql>` - SELECT - thread_id, - turn_id, - pending_message_id, - assistant_message_id, - state, - requested_at, - started_at, - completed_at, - checkpoint_turn_count, - checkpoint_ref, - checkpoint_status, - checkpoint_files_json, - source_proposed_plan_thread_id, - source_proposed_plan_id - FROM projection_turns - WHERE thread_id = ${threadId} - ORDER BY requested_at, row_id - `; - case "projection_thread_sessions": - return sql>` - SELECT * - FROM projection_thread_sessions - WHERE thread_id = ${threadId} - ORDER BY thread_id - `; - case "projection_thread_proposed_plans": - return sql>` - SELECT * - FROM projection_thread_proposed_plans - WHERE thread_id = ${threadId} - ORDER BY created_at, plan_id - `; - case "projection_pending_approvals": - return sql>` - SELECT * - FROM projection_pending_approvals - WHERE thread_id = ${threadId} - ORDER BY created_at, request_id - `; + if (options.limit !== null && events.length >= options.limit) { + break; } - })(); - return rows.pipe( - Effect.mapError((cause) => - cause instanceof Error - ? cause - : new Error(`Failed to read target checksum rows: ${String(cause)}`), - ), - ); -}; - -const checksumTables = [ - "projection_threads", - "projection_thread_messages", - "projection_thread_activities", - "projection_turns", - "projection_thread_sessions", - "projection_thread_proposed_plans", - "projection_pending_approvals", -] as const; - -function verifyTablesForMode( - mode: CliOptions["verify"], -): ReadonlyArray<(typeof checksumTables)[number]> { - switch (mode) { - case "none": - return []; - case "messages": - return ["projection_thread_messages"]; - case "full": - return checksumTables; } + return events; } -function eventTimingBucket(event: OrchestrationEventType): string { - if (classifyReplayEvent(event) === "assistant-streaming-message") { - return "thread.message-sent:assistant-streaming"; - } - return event.type; -} - -function makeTargetLayer(options: CliOptions, targetFile: string | null) { - const persistence = - options.target === "memory" - ? SqlitePersistenceMemory - : makeSqlitePersistenceLive(targetFile ?? makeDefaultTargetFile(options.threadId)); - const repositoryIdentityResolverLayer = Layer.succeed(RepositoryIdentityResolver, { - resolve: () => Effect.succeed(null), - }); - - return Layer.mergeAll( - OrchestrationProjectionPipelineLive.pipe(Layer.provideMerge(OrchestrationEventStoreLive)), - OrchestrationProjectionSnapshotQueryLive.pipe(Layer.provide(repositoryIdentityResolverLayer)), - ).pipe( - Layer.provideMerge(persistence), - Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-benchmark-" })), - Layer.provideMerge(NodeServices.layer), - ); -} - -function makeDefaultTargetFile(threadId: string): string { - return join(tmpdir(), `t3-thread-replay-${threadId}-${Date.now()}.sqlite`); -} - -function formatMs(value: number): string { - return value.toFixed(4); -} - -function printStats(label: string, timings: ReadonlyArray) { - const stats = calculateTimingStats(timings); - console.log( - `${label}: count=${stats.count} total=${formatMs(stats.totalMs)}ms mean=${formatMs( - stats.meanMs, - )}ms p50=${formatMs(stats.p50Ms)}ms p90=${formatMs(stats.p90Ms)}ms p99=${formatMs( - stats.p99Ms, - )}ms max=${formatMs(stats.maxMs)}ms`, - ); -} - -function divideForSpeedup(left: number, right: number): number { - return right === 0 ? 0 : left / right; -} - -function speedupStats( - legacy: ReturnType, - optimized: ReturnType, -) { - return { - mean: divideForSpeedup(legacy.meanMs, optimized.meanMs), - p50: divideForSpeedup(legacy.p50Ms, optimized.p50Ms), - p90: divideForSpeedup(legacy.p90Ms, optimized.p90Ms), - p99: divideForSpeedup(legacy.p99Ms, optimized.p99Ms), - }; -} - -function printCompareStats(input: { - readonly label: string; - readonly optimized: ReadonlyArray; - readonly legacy: ReadonlyArray; -}) { - const optimized = calculateTimingStats(input.optimized); - const legacy = calculateTimingStats(input.legacy); - const speedup = speedupStats(legacy, optimized); - console.log(input.label); - console.log( - ` optimized: count=${optimized.count} mean=${formatMs(optimized.meanMs)}ms p50=${formatMs( - optimized.p50Ms, - )}ms p90=${formatMs(optimized.p90Ms)}ms p99=${formatMs(optimized.p99Ms)}ms max=${formatMs( - optimized.maxMs, - )}ms`, - ); - console.log( - ` legacy: count=${legacy.count} mean=${formatMs(legacy.meanMs)}ms p50=${formatMs( - legacy.p50Ms, - )}ms p90=${formatMs(legacy.p90Ms)}ms p99=${formatMs(legacy.p99Ms)}ms max=${formatMs( - legacy.maxMs, - )}ms`, - ); - console.log( - ` speedup: mean=${speedup.mean.toFixed(1)}x p50=${speedup.p50.toFixed( - 1, - )}x p90=${speedup.p90.toFixed(1)}x p99=${speedup.p99.toFixed(1)}x`, - ); -} - -function createAssistantStreamingCompareDb(input: { - readonly thread: Record; - readonly messages?: ReadonlyArray; - readonly activities?: ReadonlyArray>; -}) { +function createCompareDb(): Database { const db = new Database(":memory:", { strict: true }); db.exec(` PRAGMA journal_mode = MEMORY; PRAGMA synchronous = OFF; + PRAGMA temp_store = MEMORY; + CREATE TABLE projection_threads ( thread_id TEXT PRIMARY KEY, - project_id TEXT, - title TEXT, + project_id TEXT NOT NULL, + title TEXT NOT NULL, + model_selection_json TEXT, + runtime_mode TEXT NOT NULL, + interaction_mode TEXT NOT NULL, branch TEXT, worktree_path TEXT, latest_turn_id TEXT, - created_at TEXT, - updated_at TEXT, - deleted_at TEXT, - runtime_mode TEXT, - interaction_mode TEXT, - model_selection_json TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, archived_at TEXT, latest_user_message_at TEXT, - pending_approval_count INTEGER DEFAULT 0, - pending_user_input_count INTEGER DEFAULT 0, - has_actionable_proposed_plan INTEGER DEFAULT 0 + pending_approval_count INTEGER NOT NULL DEFAULT 0, + pending_user_input_count INTEGER NOT NULL DEFAULT 0, + has_actionable_proposed_plan INTEGER NOT NULL DEFAULT 0, + deleted_at TEXT ); + CREATE TABLE projection_thread_messages ( message_id TEXT PRIMARY KEY, - thread_id TEXT, + thread_id TEXT NOT NULL, turn_id TEXT, - role TEXT, - text TEXT, - is_streaming INTEGER, - created_at TEXT, - updated_at TEXT, - attachments_json TEXT + role TEXT NOT NULL, + text TEXT NOT NULL, + attachments_json TEXT, + is_streaming INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL ); CREATE INDEX idx_messages_thread_created ON projection_thread_messages(thread_id, created_at, message_id); + CREATE TABLE projection_thread_activities ( activity_id TEXT PRIMARY KEY, - thread_id TEXT, + thread_id TEXT NOT NULL, turn_id TEXT, - tone TEXT, - kind TEXT, - summary TEXT, - payload_json TEXT, - created_at TEXT, - sequence INTEGER + tone TEXT NOT NULL, + kind TEXT NOT NULL, + summary TEXT NOT NULL, + payload_json TEXT NOT NULL, + sequence INTEGER, + created_at TEXT NOT NULL ); CREATE INDEX idx_activities_thread_sequence ON projection_thread_activities(thread_id, sequence, created_at, activity_id); + CREATE TABLE projection_thread_proposed_plans ( plan_id TEXT PRIMARY KEY, - thread_id TEXT, + thread_id TEXT NOT NULL, turn_id TEXT, - plan_markdown TEXT, - created_at TEXT, - updated_at TEXT, + plan_markdown TEXT NOT NULL, implemented_at TEXT, - implementation_thread_id TEXT + implementation_thread_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL ); + CREATE TABLE projection_pending_approvals ( - request_id TEXT PRIMARY KEY, - thread_id TEXT, + approval_id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL, turn_id TEXT, - status TEXT, - decision TEXT, - created_at TEXT, - resolved_at TEXT + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL ); `); + return db; +} - db.query(` - INSERT INTO projection_threads ( - thread_id, - project_id, - title, - branch, - worktree_path, - latest_turn_id, - created_at, - updated_at, - deleted_at, - runtime_mode, - interaction_mode, - model_selection_json, - archived_at, - latest_user_message_at, - pending_approval_count, - pending_user_input_count, - has_actionable_proposed_plan - ) - VALUES ( - $thread_id, - $project_id, - $title, - $branch, - $worktree_path, - $latest_turn_id, - $created_at, - $updated_at, - $deleted_at, - $runtime_mode, - $interaction_mode, - $model_selection_json, - $archived_at, - $latest_user_message_at, - $pending_approval_count, - $pending_user_input_count, - $has_actionable_proposed_plan +function readThread(db: Database, threadId: string): SourceThreadRow { + const row = db + .query( + ` + SELECT + thread_id AS threadId, + project_id AS projectId, + title, + model_selection_json AS modelSelectionJson, + runtime_mode AS runtimeMode, + interaction_mode AS interactionMode, + branch, + worktree_path AS worktreePath, + latest_turn_id AS latestTurnId, + created_at AS createdAt, + updated_at AS updatedAt, + archived_at AS archivedAt, + latest_user_message_at AS latestUserMessageAt, + pending_approval_count AS pendingApprovalCount, + pending_user_input_count AS pendingUserInputCount, + has_actionable_proposed_plan AS hasActionableProposedPlan, + deleted_at AS deletedAt + FROM projection_threads + WHERE thread_id = ? + `, ) - `).run(input.thread as SqlBindingRecord); + .get(threadId); + if (!row) { + throw new Error(`Thread ${threadId} was not found in projection_threads`); + } + return row; +} - if (input.messages !== undefined || input.activities !== undefined) { - const insertMessage = db.query(` - INSERT INTO projection_thread_messages ( - message_id, - thread_id, - turn_id, - role, - text, - is_streaming, - created_at, - updated_at, - attachments_json +function seedThread(db: Database, thread: SourceThreadRow) { + db.query( + ` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, + branch, worktree_path, latest_turn_id, created_at, updated_at, archived_at, + latest_user_message_at, pending_approval_count, pending_user_input_count, + has_actionable_proposed_plan, deleted_at ) VALUES ( - $messageId, - $threadId, - $turnId, - $role, - $text, - $isStreaming, - $createdAt, - $updatedAt, - $attachmentsJson - ) - `); - const insertActivity = db.query(` - INSERT INTO projection_thread_activities ( - activity_id, - thread_id, - turn_id, - tone, - kind, - summary, - payload_json, - created_at, - sequence + $threadId, $projectId, $title, $modelSelectionJson, $runtimeMode, $interactionMode, + $branch, $worktreePath, $latestTurnId, $createdAt, $updatedAt, $archivedAt, + $latestUserMessageAt, $pendingApprovalCount, $pendingUserInputCount, + $hasActionableProposedPlan, $deletedAt ) - VALUES ( - $activity_id, - $thread_id, - $turn_id, - $tone, - $kind, - $summary, - $payload_json, - $created_at, - $sequence - ) - `); - db.transaction(() => { - for (const message of input.messages ?? []) { - insertMessage.run(message as unknown as SqlBindingRecord); - } - for (const activity of input.activities ?? []) { - insertActivity.run(activity as SqlBindingRecord); - } - })(); + `, + ).run(thread as unknown as SqlBindingRecord); +} + +function seedLegacyProjection(source: Database, target: Database, threadId: string) { + seedThread(target, readThread(source, threadId)); + + const messages = source + .query( + ` + SELECT + message_id AS messageId, + thread_id AS threadId, + turn_id AS turnId, + role, + text, + attachments_json AS attachmentsJson, + is_streaming AS isStreaming, + created_at AS createdAt, + updated_at AS updatedAt + FROM projection_thread_messages + WHERE thread_id = ? + `, + ) + .all(threadId); + const insertMessage = target.query(` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at + ) + VALUES ( + $messageId, $threadId, $turnId, $role, $text, $attachmentsJson, $isStreaming, $createdAt, $updatedAt + ) + `); + for (const message of messages) { + insertMessage.run(message as unknown as SqlBindingRecord); } - return db; -} + const activities = source + .query( + ` + SELECT + activity_id AS activityId, + thread_id AS threadId, + turn_id AS turnId, + tone, + kind, + summary, + payload_json AS payloadJson, + sequence, + created_at AS createdAt + FROM projection_thread_activities + WHERE thread_id = ? + `, + ) + .all(threadId); + const insertActivity = target.query(` + INSERT INTO projection_thread_activities ( + activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at + ) + VALUES ( + $activityId, $threadId, $turnId, $tone, $kind, $summary, $payloadJson, $sequence, $createdAt + ) + `); + for (const activity of activities) { + insertActivity.run(activity as unknown as SqlBindingRecord); + } -function selectSampledWindowEvents( - events: ReadonlyArray, - sampleSize: number, -): ReadonlyArray { - if (events.length <= sampleSize) { - return events; + const plans = source + .query( + ` + SELECT + plan_id AS planId, + thread_id AS threadId, + turn_id AS turnId, + plan_markdown AS planMarkdown, + implemented_at AS implementedAt, + implementation_thread_id AS implementationThreadId, + created_at AS createdAt, + updated_at AS updatedAt + FROM projection_thread_proposed_plans + WHERE thread_id = ? + `, + ) + .all(threadId); + const insertPlan = target.query(` + INSERT INTO projection_thread_proposed_plans ( + plan_id, thread_id, turn_id, plan_markdown, implemented_at, implementation_thread_id, created_at, updated_at + ) + VALUES ( + $planId, $threadId, $turnId, $planMarkdown, $implementedAt, $implementationThreadId, $createdAt, $updatedAt + ) + `); + for (const plan of plans) { + insertPlan.run(plan as unknown as SqlBindingRecord); } - const step = events.length / sampleSize; - const selected: Array = []; - for (let index = 0; index < sampleSize; index += 1) { - selected.push(events[Math.floor(index * step)]!); + + const approvals = source + .query( + ` + SELECT + request_id AS approvalId, + thread_id AS threadId, + turn_id AS turnId, + status, + created_at AS createdAt, + COALESCE(resolved_at, created_at) AS updatedAt + FROM projection_pending_approvals + WHERE thread_id = ? + `, + ) + .all(threadId); + const insertApproval = target.query(` + INSERT INTO projection_pending_approvals ( + approval_id, thread_id, turn_id, status, created_at, updated_at + ) + VALUES ( + $approvalId, $threadId, $turnId, $status, $createdAt, $updatedAt + ) + `); + for (const approval of approvals) { + insertApproval.run(approval as unknown as SqlBindingRecord); } - return selected; } -function runAssistantStreamingCompare(options: CliOptions) { - const events = readAssistantStreamingEvents(options.sourceDb, options.threadId).slice( - 0, - options.limit ?? undefined, - ); - const thread = readSourceThreadRow(options.sourceDb, options.threadId); - const sourceMessages = readSourceMessageRows(options.sourceDb, options.threadId); - const sourceActivities = readSourceActivityRows(options.sourceDb, options.threadId); - const optimizedDb = createAssistantStreamingCompareDb({ thread }); - const optimizedAppend = optimizedDb.query(` +function seedOptimizedProjection(source: Database, target: Database, threadId: string) { + seedThread(target, readThread(source, threadId)); +} + +function makeOptimizedRunner(db: Database) { + const append = db.query(` INSERT INTO projection_thread_messages ( - message_id, - thread_id, - turn_id, - role, - text, - is_streaming, - created_at, - updated_at, - attachments_json + message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at ) VALUES ( - $messageId, - $threadId, - $turnId, - $role, - $text, - 1, - $createdAt, - $updatedAt, - NULL + $messageId, $threadId, $turnId, $role, $text, NULL, 1, $createdAt, $updatedAt ) - ON CONFLICT(message_id) DO UPDATE SET + ON CONFLICT (message_id) + DO UPDATE SET thread_id = excluded.thread_id, turn_id = excluded.turn_id, role = excluded.role, @@ -1123,448 +520,279 @@ function runAssistantStreamingCompare(options: CliOptions) { is_streaming = excluded.is_streaming, updated_at = excluded.updated_at `); - - const legacyDb = createAssistantStreamingCompareDb({ - thread, - messages: sourceMessages, - activities: sourceActivities, - }); - const legacyGet = legacyDb.query< - { readonly text: string; readonly createdAt: string; readonly attachmentsJson: string | null }, - [string] - >(` - SELECT - text, - created_at AS createdAt, - attachments_json AS attachmentsJson - FROM projection_thread_messages - WHERE message_id = ? + const touchThread = db.query(` + UPDATE projection_threads + SET updated_at = $updatedAt + WHERE thread_id = $threadId `); - const legacyUpsert = legacyDb.query(` + + return (event: AssistantStreamingEvent): number => { + const started = performance.now(); + append.run({ + messageId: event.messageId, + threadId: event.threadId, + turnId: event.turnId, + role: event.role, + text: event.text, + createdAt: event.createdAt, + updatedAt: event.updatedAt, + }); + touchThread.run({ threadId: event.threadId, updatedAt: event.updatedAt }); + return performance.now() - started; + }; +} + +function makeLegacyRunner(db: Database) { + const getExistingText = db.query<{ readonly text: string }, [string]>( + `SELECT text FROM projection_thread_messages WHERE message_id = ?`, + ); + const upsertFullText = db.query(` INSERT INTO projection_thread_messages ( - message_id, - thread_id, - turn_id, - role, - text, - is_streaming, - created_at, - updated_at, - attachments_json + message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at ) VALUES ( - $messageId, - $threadId, - $turnId, - $role, - $nextText, - 1, - $createdAtForWrite, - $updatedAt, - $attachmentsJson + $messageId, $threadId, $turnId, $role, $text, NULL, 1, $createdAt, $updatedAt ) - ON CONFLICT(message_id) DO UPDATE SET + ON CONFLICT (message_id) + DO UPDATE SET thread_id = excluded.thread_id, turn_id = excluded.turn_id, role = excluded.role, text = excluded.text, is_streaming = excluded.is_streaming, - updated_at = excluded.updated_at, - attachments_json = COALESCE( - excluded.attachments_json, - projection_thread_messages.attachments_json - ) + updated_at = excluded.updated_at `); - const updateThread = legacyDb.query(` + const touchThread = db.query(` UPDATE projection_threads - SET updated_at = ? - WHERE thread_id = ? + SET updated_at = $updatedAt + WHERE thread_id = $threadId `); - const listMessages = legacyDb.query< - { readonly role: string; readonly createdAt: string }, - [string] - >(` - SELECT role, created_at AS createdAt + const listMessages = db.query(` + SELECT message_id, role, text, attachments_json, is_streaming, created_at, updated_at FROM projection_thread_messages WHERE thread_id = ? ORDER BY created_at ASC, message_id ASC `); - const listActivities = legacyDb.query< - { readonly kind: string; readonly payloadJson: string; readonly createdAt: string }, - [string] - >(` - SELECT - kind, - payload_json AS payloadJson, - created_at AS createdAt + const listActivities = db.query(` + SELECT activity_id, turn_id, tone, kind, summary, payload_json, sequence, created_at FROM projection_thread_activities WHERE thread_id = ? ORDER BY - CASE WHEN sequence IS NULL THEN 0 ELSE 1 END, + CASE WHEN sequence IS NULL THEN 0 ELSE 1 END ASC, sequence ASC, created_at ASC, activity_id ASC `); - const countApprovals = legacyDb.query<{ readonly count: number }, [string]>(` - SELECT count(*) AS count - FROM projection_pending_approvals - WHERE thread_id = ? - AND status = 'pending' - `); - const listPlans = legacyDb.query< - { - readonly planId: string; - readonly turnId: string | null; - readonly implementedAt: string | null; - }, - [string] - >(` - SELECT - plan_id AS planId, - turn_id AS turnId, - implemented_at AS implementedAt + const listPlans = db.query(` + SELECT plan_id, turn_id, plan_markdown, implemented_at, implementation_thread_id, created_at, updated_at FROM projection_thread_proposed_plans WHERE thread_id = ? - ORDER BY updated_at ASC, plan_id ASC + ORDER BY created_at ASC, plan_id ASC `); + const pendingApprovals = db.query(` + SELECT approval_id + FROM projection_pending_approvals + WHERE thread_id = ? AND status = 'pending' + `); + + return (event: AssistantStreamingEvent): number => { + const started = performance.now(); + db.exec("BEGIN"); + try { + const existing = getExistingText.get(event.messageId); + const nextText = `${existing?.text ?? ""}${event.text}`; + upsertFullText.run({ + messageId: event.messageId, + threadId: event.threadId, + turnId: event.turnId, + role: event.role, + text: nextText, + createdAt: event.createdAt, + updatedAt: event.updatedAt, + }); + touchThread.run({ threadId: event.threadId, updatedAt: event.updatedAt }); + + // This mirrors the old per-delta shell summary refresh shape: hydrate the + // thread's message/activity/plan/approval read-model state after every token. + const messages = listMessages.all(event.threadId) as ReadonlyArray<{ + readonly role: string; + readonly created_at: string; + }>; + const activities = listActivities.all(event.threadId) as ReadonlyArray<{ + readonly kind: string; + readonly payload_json: string; + }>; + const plans = listPlans.all(event.threadId); + const approvals = pendingApprovals.all(event.threadId); + const latestUserMessageAt = messages + .filter((message) => message.role === "user") + .map((message) => message.created_at) + .toSorted() + .at(-1); + const pendingUserInputCount = activities.filter((activity) => { + if (activity.kind !== "input") { + return false; + } + try { + const payload = JSON.parse(activity.payload_json) as { readonly state?: string }; + return payload.state === "pending"; + } catch { + return false; + } + }).length; + const derivedSummary = { + latestUserMessageAt, + pendingUserInputCount, + planCount: plans.length, + pendingApprovalCount: approvals.length, + }; + _summaryRefreshSink = JSON.stringify(derivedSummary); + + db.exec("ROLLBACK"); + return performance.now() - started; + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + }; +} + +function selectLegacyEvents( + events: ReadonlyArray, + options: CliOptions, +): ReadonlyArray { + if (options.legacyMode === "full" || events.length <= options.legacySamplePerWindow) { + return events; + } + + const selected: Array = []; + const step = events.length / options.legacySamplePerWindow; + for (let index = 0; index < options.legacySamplePerWindow; index += 1) { + const event = events[Math.min(events.length - 1, Math.floor(index * step))]; + if (event) { + selected.push(event); + } + } + return selected; +} + +function formatMs(value: number): string { + return value.toFixed(4); +} + +function divideForSpeedup(left: number, right: number): number { + return right === 0 ? 0 : left / right; +} + +function printCompareStats(input: { + readonly label: string; + readonly optimized: ReadonlyArray; + readonly legacy: ReadonlyArray; +}) { + const optimized = calculateTimingStats(input.optimized); + const legacy = calculateTimingStats(input.legacy); + console.log(input.label); + console.log( + ` optimized: count=${optimized.count} mean=${formatMs(optimized.meanMs)}ms p50=${formatMs( + optimized.p50Ms, + )}ms p90=${formatMs(optimized.p90Ms)}ms p99=${formatMs(optimized.p99Ms)}ms max=${formatMs( + optimized.maxMs, + )}ms`, + ); + console.log( + ` legacy: count=${legacy.count} mean=${formatMs(legacy.meanMs)}ms p50=${formatMs( + legacy.p50Ms, + )}ms p90=${formatMs(legacy.p90Ms)}ms p99=${formatMs(legacy.p99Ms)}ms max=${formatMs( + legacy.maxMs, + )}ms`, + ); + console.log( + ` speedup: mean=${formatMs(divideForSpeedup(legacy.meanMs, optimized.meanMs))}x p50=${formatMs( + divideForSpeedup(legacy.p50Ms, optimized.p50Ms), + )}x p90=${formatMs(divideForSpeedup(legacy.p90Ms, optimized.p90Ms))}x p99=${formatMs( + divideForSpeedup(legacy.p99Ms, optimized.p99Ms), + )}x`, + ); +} + +function runBenchmark(options: CliOptions) { + const source = openSourceDb(options.sourceDb); + const events = readAssistantStreamingEvents(source, options); + if (events.length === 0) { + throw new Error(`No assistant streaming events found for thread ${options.threadId}`); + } + + const optimizedDb = createCompareDb(); + const legacyDb = createCompareDb(); + seedOptimizedProjection(source, optimizedDb, options.threadId); + seedLegacyProjection(source, legacyDb, options.threadId); - const optimizedAll: Array = []; - const legacyAll: Array = []; + const runOptimized = makeOptimizedRunner(optimizedDb); + const runLegacy = makeLegacyRunner(legacyDb); + const optimizedTimings: Array = []; + const legacyTimings: Array = []; const windows: Array<{ - readonly label: string; + readonly from: number; + readonly to: number; readonly optimized: ReadonlyArray; readonly legacy: ReadonlyArray; - readonly optimizedCount: number; - readonly legacyCount: number; }> = []; for (let start = 0; start < events.length; start += options.windowSize) { const end = Math.min(events.length, start + options.windowSize); const windowEvents = events.slice(start, end); - const optimizedWindow: Array = []; + const windowOptimized: Array = []; + const windowLegacy: Array = []; + for (const event of windowEvents) { - const startedAt = performance.now(); - optimizedAppend.run(event as unknown as SqlBindingRecord); - const elapsed = performance.now() - startedAt; - optimizedWindow.push(elapsed); - optimizedAll.push(elapsed); + const timing = runOptimized(event); + windowOptimized.push(timing); + optimizedTimings.push(timing); } - const legacyWindowEvents = - options.legacyMode === "full" - ? windowEvents - : selectSampledWindowEvents(windowEvents, options.legacySamplePerWindow); - const legacyWindow: Array = []; - for (const event of legacyWindowEvents) { - legacyDb.exec("BEGIN"); - const startedAt = performance.now(); - const existing = legacyGet.get(event.messageId); - const nextText = existing === null ? event.text : `${existing.text}${event.text}`; - legacyUpsert.run({ - ...event, - nextText, - attachmentsJson: existing?.attachmentsJson ?? null, - createdAtForWrite: existing?.createdAt ?? event.createdAt, - } as unknown as SqlBindingRecord); - updateThread.run(event.updatedAt, event.threadId); - const messages = listMessages.all(event.threadId); - const activities = listActivities.all(event.threadId); - const approvals = countApprovals.get(event.threadId)?.count ?? 0; - const plans = listPlans.all(event.threadId); - let latestUserMessageAt: string | null = null; - for (const message of messages) { - if (message.role === "user") { - latestUserMessageAt = message.createdAt; - } - } - let pendingUserInputCount = 0; - for (const activity of activities) { - if (activity.kind === "user-input.requested") { - pendingUserInputCount += 1; - } - } - void approvals; - void plans; - void latestUserMessageAt; - void pendingUserInputCount; - const elapsed = performance.now() - startedAt; - legacyWindow.push(elapsed); - legacyAll.push(elapsed); - legacyDb.exec("ROLLBACK"); + for (const event of selectLegacyEvents(windowEvents, options)) { + const timing = runLegacy(event); + windowLegacy.push(timing); + legacyTimings.push(timing); } windows.push({ - label: `window ${start + 1}-${end}`, - optimized: optimizedWindow, - legacy: legacyWindow, - optimizedCount: windowEvents.length, - legacyCount: legacyWindowEvents.length, + from: start + 1, + to: end, + optimized: windowOptimized, + legacy: windowLegacy, }); console.log( - `compare progress ${end}/${events.length}: optimized=${windowEvents.length} legacy=${legacyWindowEvents.length}`, + `compare progress ${end}/${events.length}: optimized=${windowOptimized.length} legacy=${windowLegacy.length}`, ); } console.log(`thread=${options.threadId}`); console.log(`sourceDb=${options.sourceDb}`); - console.log("mode=compare assistant-streaming"); + console.log("mode=assistant-streaming"); console.log(`legacyMode=${options.legacyMode}`); console.log(`windowSize=${options.windowSize}`); console.log(`assistantStreamingEvents=${events.length}`); - console.log(`legacySamplePerWindow=${options.legacySamplePerWindow}`); + if (options.legacyMode === "sampled") { + console.log(`legacySamplePerWindow=${options.legacySamplePerWindow}`); + } printCompareStats({ label: "overall", - optimized: optimizedAll, - legacy: legacyAll, + optimized: optimizedTimings, + legacy: legacyTimings, }); for (const window of windows) { printCompareStats({ - label: `${window.label} optimizedEvents=${window.optimizedCount} legacyEvents=${window.legacyCount}`, + label: `window ${window.from}-${window.to} optimizedEvents=${window.optimized.length} legacyEvents=${window.legacy.length}`, optimized: window.optimized, legacy: window.legacy, }); } - - optimizedDb.close(); - legacyDb.close(); -} - -const runReplay = (options: CliOptions, targetFile: string | null) => - Effect.gen(function* () { - const sourceEvents = readSourceEvents(options.sourceDb, options.threadId).slice( - 0, - options.limit ?? undefined, - ); - const sourceProject = readSourceProject(options.sourceDb, options.threadId); - if (sourceEvents.length === 0) { - throw new Error(`No thread events found for ${options.threadId}`); - } - - const sql = yield* SqlClient.SqlClient; - if (sourceProject !== null) { - yield* sql` - INSERT INTO projection_projects ( - project_id, - title, - workspace_root, - scripts_json, - created_at, - updated_at, - deleted_at, - default_model_selection_json - ) - VALUES ( - ${sourceProject.projectId}, - ${sourceProject.title}, - ${sourceProject.workspaceRoot}, - ${sourceProject.scriptsJson}, - ${sourceProject.createdAt}, - ${sourceProject.updatedAt}, - ${sourceProject.deletedAt}, - ${sourceProject.defaultModelSelectionJson} - ) - `; - } - - const projectionPipeline = yield* OrchestrationProjectionPipeline; - const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; - const allTimings: Array = []; - const assistantStreamingTimings: Array = []; - const otherTimings: Array = []; - const timingsByBucket = new Map>(); - const startedAt = performance.now(); - - for (const [index, event] of sourceEvents.entries()) { - const eventStartedAt = performance.now(); - yield* projectionPipeline.projectEvent(event); - const elapsedMs = performance.now() - eventStartedAt; - allTimings.push(elapsedMs); - const bucket = eventTimingBucket(event); - const bucketTimings = timingsByBucket.get(bucket); - if (bucketTimings === undefined) { - timingsByBucket.set(bucket, [elapsedMs]); - } else { - bucketTimings.push(elapsedMs); - } - if (classifyReplayEvent(event) === "assistant-streaming-message") { - assistantStreamingTimings.push(elapsedMs); - } else { - otherTimings.push(elapsedMs); - } - const replayed = index + 1; - if (replayed % options.progressEvery === 0 || replayed === sourceEvents.length) { - const stats = calculateTimingStats(allTimings.slice(-options.progressEvery)); - console.log( - `progress ${replayed}/${sourceEvents.length}: recent_mean=${formatMs( - stats.meanMs, - )}ms recent_p50=${formatMs(stats.p50Ms)}ms recent_p90=${formatMs( - stats.p90Ms, - )}ms recent_p99=${formatMs(stats.p99Ms)}ms`, - ); - } - } - - const totalWallMs = performance.now() - startedAt; - const detail = yield* projectionSnapshotQuery.getThreadDetailById( - ThreadId.make(options.threadId), - ); - if (Option.isNone(detail)) { - throw new Error("Replay completed but thread detail was not projected"); - } - - const targetChecksums: Record = {}; - const sourceChecksums: Record = {}; - const tablesToVerify = options.limit === null ? verifyTablesForMode(options.verify) : []; - if (tablesToVerify.length > 0) { - for (const table of tablesToVerify) { - const rows = yield* readTargetChecksumRows(sql, table, options.threadId); - targetChecksums[table] = checksumRows(rows); - } - - for (const table of tablesToVerify) { - sourceChecksums[table] = checksumRows( - readSourceChecksumRows(options.sourceDb, table, options.threadId), - ); - } - } - - return { - sourceEvents, - allTimings, - assistantStreamingTimings, - otherTimings, - timingsByBucket, - totalWallMs, - sourceChecksums, - targetChecksums, - detail: detail.value, - }; - }).pipe(Effect.provide(makeTargetLayer(options, targetFile))); - -function main() { - const options = parseArgs(process.argv.slice(2)); - if (options.compare === "assistant-streaming") { - runAssistantStreamingCompare(options); - return; - } - - if (options.diffOnly) { - const sourceEvents = readSourceEvents(options.sourceDb, options.threadId).slice( - 0, - options.limit ?? undefined, - ); - const sourceRows = readSourceMessageRows(options.sourceDb, options.threadId); - const expectedRows = deriveMessageRowsFromEvents(sourceEvents); - const diffs = buildMessageDiffs({ - sourceRows, - expectedRows, - limit: options.diffMessages > 0 ? options.diffMessages : 20, - }); - console.log(`thread=${options.threadId}`); - console.log(`sourceDb=${options.sourceDb}`); - console.log(`mode=diff-only`); - console.log(`events=${sourceEvents.length}`); - console.log(`sourceMessages=${sourceRows.length}`); - console.log(`eventDerivedMessages=${expectedRows.length}`); - console.log(`message_diffs=${diffs.length}`); - for (const diff of diffs) { - console.log(` ${JSON.stringify(diff)}`); - } - return; - } - - const targetFile = - options.target === "file" - ? resolve(options.targetFile ?? makeDefaultTargetFile(options.threadId)) - : null; - if (targetFile !== null && existsSync(targetFile)) { - rmSync(targetFile, { force: true }); - } - - const result = Effect.runPromise( - runReplay(options, targetFile).pipe( - Effect.mapError((cause) => - cause instanceof Error ? cause : new Error(`Replay benchmark failed: ${String(cause)}`), - ), - ), - ); - result - .then((replay) => { - console.log(`thread=${options.threadId}`); - console.log(`sourceDb=${options.sourceDb}`); - console.log(`target=${options.target}${targetFile ? `:${targetFile}` : ""}`); - console.log(`events=${replay.sourceEvents.length}`); - console.log(`messages=${replay.detail.messages.length}`); - console.log(`activities=${replay.detail.activities.length}`); - console.log(`wall=${formatMs(replay.totalWallMs)}ms`); - printStats("all", replay.allTimings); - printStats("assistant_streaming", replay.assistantStreamingTimings); - printStats("other", replay.otherTimings); - console.log("event_type_buckets:"); - const sortedBuckets = [...replay.timingsByBucket.entries()].toSorted( - ([, left], [, right]) => right.length - left.length, - ); - for (const [bucket, timings] of sortedBuckets) { - printStats(` ${bucket}`, timings); - } - - console.log("samples:"); - for (const sample of buildTimingSamples(replay.allTimings, options.sampleEvery)) { - console.log( - ` ${sample.fromEvent}-${sample.toEvent}: mean=${formatMs( - sample.stats.meanMs, - )}ms p50=${formatMs(sample.stats.p50Ms)}ms p90=${formatMs( - sample.stats.p90Ms, - )}ms p99=${formatMs(sample.stats.p99Ms)}ms max=${formatMs(sample.stats.maxMs)}ms`, - ); - } - - const tablesToVerify = options.limit === null ? verifyTablesForMode(options.verify) : []; - if (tablesToVerify.length > 0) { - console.log(`checksums (${options.verify}):`); - let mismatchCount = 0; - for (const table of tablesToVerify) { - const source = replay.sourceChecksums[table]; - const target = replay.targetChecksums[table]; - const ok = source === target; - if (!ok) { - mismatchCount += 1; - } - console.log(` ${table}: ${ok ? "ok" : "mismatch"}`); - } - if (mismatchCount > 0) { - process.exitCode = 1; - } - } else if (options.verify === "none") { - console.log("checksums: skipped by --verify none"); - } else { - console.log("checksums: skipped for limited replay"); - } - if (options.diffMessages > 0) { - const sourceRows = readSourceMessageRows(options.sourceDb, options.threadId); - const expectedRows = deriveMessageRowsFromEvents(replay.sourceEvents); - const diffs = buildMessageDiffs({ - sourceRows, - expectedRows, - limit: options.diffMessages, - }); - console.log(`message_diffs: ${diffs.length}`); - for (const diff of diffs) { - console.log(` ${JSON.stringify(diff)}`); - } - } - if (targetFile !== null && !options.keepTarget) { - rmSync(targetFile, { force: true }); - rmSync(`${targetFile}-shm`, { force: true }); - rmSync(`${targetFile}-wal`, { force: true }); - } - }) - .catch((cause: unknown) => { - console.error(cause); - process.exitCode = 1; - }); } -if (import.meta.main) { - main(); +try { + runBenchmark(parseArgs(process.argv.slice(2))); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); } diff --git a/apps/server/scripts/replay_thread/core.test.ts b/apps/server/scripts/replay_thread/core.test.ts index 56c940d209..f0ff1849db 100644 --- a/apps/server/scripts/replay_thread/core.test.ts +++ b/apps/server/scripts/replay_thread/core.test.ts @@ -1,12 +1,6 @@ import { assert, it } from "@effect/vitest"; -import { - buildTimingSamples, - calculateTimingStats, - checksumRows, - classifyReplayEvent, - stableJson, -} from "./core.ts"; +import { buildTimingSamples, calculateTimingStats } from "./core.ts"; it("calculates timing stats with stable percentile boundaries", () => { assert.deepEqual(calculateTimingStats([]), { @@ -49,29 +43,3 @@ it("builds contiguous timing samples", () => { }, ]); }); - -it("hashes rows independently of object key insertion order", () => { - const left = [{ b: 2, a: { d: 4, c: 3 } }]; - const right = [{ a: { c: 3, d: 4 }, b: 2 }]; - - assert.equal(stableJson(left), stableJson(right)); - assert.equal(checksumRows(left), checksumRows(right)); -}); - -it("classifies assistant streaming events separately from other events", () => { - assert.equal( - classifyReplayEvent({ - type: "thread.message-sent", - payload: { role: "assistant", streaming: true }, - }), - "assistant-streaming-message", - ); - assert.equal( - classifyReplayEvent({ - type: "thread.message-sent", - payload: { role: "assistant", streaming: false }, - }), - "other", - ); - assert.equal(classifyReplayEvent({ type: "thread.created", payload: {} }), "other"); -}); diff --git a/apps/server/scripts/replay_thread/core.ts b/apps/server/scripts/replay_thread/core.ts index e714e64e80..ac96b833be 100644 --- a/apps/server/scripts/replay_thread/core.ts +++ b/apps/server/scripts/replay_thread/core.ts @@ -1,5 +1,3 @@ -import { createHash } from "node:crypto"; - export interface TimingStats { readonly count: number; readonly totalMs: number; @@ -63,46 +61,3 @@ export function buildTimingSamples( } return samples; } - -function stableValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(stableValue); - } - if (typeof value === "object" && value !== null) { - return Object.fromEntries( - Object.entries(value) - .toSorted(([left], [right]) => left.localeCompare(right)) - .map(([key, nested]) => [key, stableValue(nested)]), - ); - } - return value; -} - -export function stableJson(value: unknown): string { - return JSON.stringify(stableValue(value)); -} - -export function checksumRows(rows: ReadonlyArray>): string { - const hash = createHash("sha256"); - for (const row of rows) { - hash.update(stableJson(row)); - hash.update("\n"); - } - return hash.digest("hex"); -} - -export function classifyReplayEvent(event: { - readonly type: string; - readonly payload: unknown; -}): "assistant-streaming-message" | "other" { - if (event.type !== "thread.message-sent") { - return "other"; - } - const payload = - typeof event.payload === "object" && event.payload !== null - ? (event.payload as Record) - : {}; - return payload.role === "assistant" && payload.streaming === true - ? "assistant-streaming-message" - : "other"; -} From 599c1aa8d16a96cca74045baa9328e4befe78135 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Sun, 10 May 2026 18:05:50 +0100 Subject: [PATCH 06/10] Remove replay benchmark tooling --- .../server/scripts/replay_thread/benchmark.ts | 798 ------------------ .../server/scripts/replay_thread/core.test.ts | 45 - apps/server/scripts/replay_thread/core.ts | 63 -- 3 files changed, 906 deletions(-) delete mode 100644 apps/server/scripts/replay_thread/benchmark.ts delete mode 100644 apps/server/scripts/replay_thread/core.test.ts delete mode 100644 apps/server/scripts/replay_thread/core.ts diff --git a/apps/server/scripts/replay_thread/benchmark.ts b/apps/server/scripts/replay_thread/benchmark.ts deleted file mode 100644 index 70d85e885f..0000000000 --- a/apps/server/scripts/replay_thread/benchmark.ts +++ /dev/null @@ -1,798 +0,0 @@ -import { existsSync } from "node:fs"; -import { performance } from "node:perf_hooks"; - -import { Database } from "bun:sqlite"; - -import { calculateTimingStats } from "./core.ts"; - -interface CliOptions { - readonly sourceDb: string; - readonly threadId: string; - readonly windowSize: number; - readonly legacyMode: "sampled" | "full"; - readonly legacySamplePerWindow: number; - readonly limit: number | null; -} - -interface SourceEventRow { - readonly sequence: number; - readonly payloadJson: string; -} - -interface AssistantStreamingEvent { - readonly sequence: number; - readonly messageId: string; - readonly threadId: string; - readonly turnId: string | null; - readonly role: "assistant"; - readonly text: string; - readonly createdAt: string; - readonly updatedAt: string; -} - -interface SourceThreadRow { - readonly threadId: string; - readonly projectId: string; - readonly title: string; - readonly modelSelectionJson: string | null; - readonly runtimeMode: string; - readonly interactionMode: string; - readonly branch: string | null; - readonly worktreePath: string | null; - readonly latestTurnId: string | null; - readonly createdAt: string; - readonly updatedAt: string; - readonly archivedAt: string | null; - readonly latestUserMessageAt: string | null; - readonly pendingApprovalCount: number; - readonly pendingUserInputCount: number; - readonly hasActionableProposedPlan: number; - readonly deletedAt: string | null; -} - -interface SourceMessageRow { - readonly messageId: string; - readonly threadId: string; - readonly turnId: string | null; - readonly role: string; - readonly text: string; - readonly attachmentsJson: string | null; - readonly isStreaming: number; - readonly createdAt: string; - readonly updatedAt: string; -} - -interface SourceActivityRow { - readonly activityId: string; - readonly threadId: string; - readonly turnId: string | null; - readonly tone: string; - readonly kind: string; - readonly summary: string; - readonly payloadJson: string; - readonly sequence: number | null; - readonly createdAt: string; -} - -interface SourcePlanRow { - readonly planId: string; - readonly threadId: string; - readonly turnId: string | null; - readonly planMarkdown: string; - readonly implementedAt: string | null; - readonly implementationThreadId: string | null; - readonly createdAt: string; - readonly updatedAt: string; -} - -interface SourceApprovalRow { - readonly approvalId: string; - readonly threadId: string; - readonly turnId: string | null; - readonly status: string; - readonly createdAt: string; - readonly updatedAt: string; -} - -type SqlBindingRecord = Record; - -let _summaryRefreshSink = ""; - -function parseArgs(argv: ReadonlyArray): CliOptions { - let sourceDb: string | undefined; - let threadId: string | undefined; - let windowSize = 10_000; - let legacyMode: CliOptions["legacyMode"] = "sampled"; - let legacySamplePerWindow = 500; - let limit: number | null = null; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - const next = () => { - const value = argv[index + 1]; - if (value === undefined) { - throw new Error(`Missing value for ${arg}`); - } - index += 1; - return value; - }; - - switch (arg) { - case "--source-db": - sourceDb = next(); - break; - case "--thread-id": - threadId = next(); - break; - case "--window-size": - windowSize = Number(next()); - break; - case "--legacy-mode": { - const value = next(); - if (value !== "sampled" && value !== "full") { - throw new Error("--legacy-mode must be sampled or full"); - } - legacyMode = value; - break; - } - case "--legacy-sample-per-window": - legacySamplePerWindow = Number(next()); - break; - case "--limit": - limit = Number(next()); - break; - case "--help": - case "-h": - printUsage(); - process.exit(0); - default: - throw new Error(`Unknown argument: ${arg}`); - } - } - - if (!sourceDb) { - throw new Error("Missing --source-db"); - } - if (!threadId) { - throw new Error("Missing --thread-id"); - } - if (!Number.isFinite(windowSize) || windowSize < 1) { - throw new Error("--window-size must be a positive number"); - } - if (!Number.isFinite(legacySamplePerWindow) || legacySamplePerWindow < 1) { - throw new Error("--legacy-sample-per-window must be a positive number"); - } - if (limit !== null && (!Number.isFinite(limit) || limit < 1)) { - throw new Error("--limit must be a positive number"); - } - - return { - sourceDb, - threadId, - windowSize, - legacyMode, - legacySamplePerWindow, - limit, - }; -} - -function printUsage() { - console.log(`Usage: - bun apps/server/scripts/replay_thread/benchmark.ts \\ - --source-db C:\\Users\\mike\\.t3\\dev\\state.sqlite \\ - --thread-id de1c398f-5d3c-40e4-911c-2b672653cda7 \\ - --window-size 10000 \\ - --legacy-sample-per-window 500 - -Options: - --source-db SQLite DB containing the copied production thread - --thread-id Thread to benchmark - --window-size Assistant-streaming events per output window (default: 10000) - --legacy-mode sampled|full Sample old path per window, or run every event (default: sampled) - --legacy-sample-per-window Old-path samples per window in sampled mode (default: 500) - --limit Optional cap for smoke runs -`); -} - -function openSourceDb(path: string): Database { - if (!existsSync(path)) { - throw new Error(`Source DB does not exist: ${path}`); - } - return new Database(path, { readonly: true, strict: true }); -} - -function readAssistantStreamingEvents( - db: Database, - options: CliOptions, -): ReadonlyArray { - const rows = db - .query( - ` - SELECT sequence, payload_json AS payloadJson - FROM orchestration_events - WHERE stream_id = ? AND event_type = 'thread.message-sent' - ORDER BY sequence ASC - `, - ) - .all(options.threadId); - - const events: Array = []; - for (const row of rows) { - const payload = JSON.parse(row.payloadJson) as Partial & { - readonly streaming?: unknown; - }; - if ( - payload.threadId !== options.threadId || - payload.role !== "assistant" || - payload.streaming !== true || - typeof payload.messageId !== "string" || - typeof payload.text !== "string" || - typeof payload.createdAt !== "string" || - typeof payload.updatedAt !== "string" - ) { - continue; - } - events.push({ - sequence: row.sequence, - messageId: payload.messageId, - threadId: options.threadId, - turnId: typeof payload.turnId === "string" ? payload.turnId : null, - role: "assistant", - text: payload.text, - createdAt: payload.createdAt, - updatedAt: payload.updatedAt, - }); - if (options.limit !== null && events.length >= options.limit) { - break; - } - } - return events; -} - -function createCompareDb(): Database { - const db = new Database(":memory:", { strict: true }); - db.exec(` - PRAGMA journal_mode = MEMORY; - PRAGMA synchronous = OFF; - PRAGMA temp_store = MEMORY; - - CREATE TABLE projection_threads ( - thread_id TEXT PRIMARY KEY, - project_id TEXT NOT NULL, - title TEXT NOT NULL, - model_selection_json TEXT, - runtime_mode TEXT NOT NULL, - interaction_mode TEXT NOT NULL, - branch TEXT, - worktree_path TEXT, - latest_turn_id TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - archived_at TEXT, - latest_user_message_at TEXT, - pending_approval_count INTEGER NOT NULL DEFAULT 0, - pending_user_input_count INTEGER NOT NULL DEFAULT 0, - has_actionable_proposed_plan INTEGER NOT NULL DEFAULT 0, - deleted_at TEXT - ); - - CREATE TABLE projection_thread_messages ( - message_id TEXT PRIMARY KEY, - thread_id TEXT NOT NULL, - turn_id TEXT, - role TEXT NOT NULL, - text TEXT NOT NULL, - attachments_json TEXT, - is_streaming INTEGER NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE INDEX idx_messages_thread_created - ON projection_thread_messages(thread_id, created_at, message_id); - - CREATE TABLE projection_thread_activities ( - activity_id TEXT PRIMARY KEY, - thread_id TEXT NOT NULL, - turn_id TEXT, - tone TEXT NOT NULL, - kind TEXT NOT NULL, - summary TEXT NOT NULL, - payload_json TEXT NOT NULL, - sequence INTEGER, - created_at TEXT NOT NULL - ); - CREATE INDEX idx_activities_thread_sequence - ON projection_thread_activities(thread_id, sequence, created_at, activity_id); - - CREATE TABLE projection_thread_proposed_plans ( - plan_id TEXT PRIMARY KEY, - thread_id TEXT NOT NULL, - turn_id TEXT, - plan_markdown TEXT NOT NULL, - implemented_at TEXT, - implementation_thread_id TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - - CREATE TABLE projection_pending_approvals ( - approval_id TEXT PRIMARY KEY, - thread_id TEXT NOT NULL, - turn_id TEXT, - status TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - `); - return db; -} - -function readThread(db: Database, threadId: string): SourceThreadRow { - const row = db - .query( - ` - SELECT - thread_id AS threadId, - project_id AS projectId, - title, - model_selection_json AS modelSelectionJson, - runtime_mode AS runtimeMode, - interaction_mode AS interactionMode, - branch, - worktree_path AS worktreePath, - latest_turn_id AS latestTurnId, - created_at AS createdAt, - updated_at AS updatedAt, - archived_at AS archivedAt, - latest_user_message_at AS latestUserMessageAt, - pending_approval_count AS pendingApprovalCount, - pending_user_input_count AS pendingUserInputCount, - has_actionable_proposed_plan AS hasActionableProposedPlan, - deleted_at AS deletedAt - FROM projection_threads - WHERE thread_id = ? - `, - ) - .get(threadId); - if (!row) { - throw new Error(`Thread ${threadId} was not found in projection_threads`); - } - return row; -} - -function seedThread(db: Database, thread: SourceThreadRow) { - db.query( - ` - INSERT INTO projection_threads ( - thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, - branch, worktree_path, latest_turn_id, created_at, updated_at, archived_at, - latest_user_message_at, pending_approval_count, pending_user_input_count, - has_actionable_proposed_plan, deleted_at - ) - VALUES ( - $threadId, $projectId, $title, $modelSelectionJson, $runtimeMode, $interactionMode, - $branch, $worktreePath, $latestTurnId, $createdAt, $updatedAt, $archivedAt, - $latestUserMessageAt, $pendingApprovalCount, $pendingUserInputCount, - $hasActionableProposedPlan, $deletedAt - ) - `, - ).run(thread as unknown as SqlBindingRecord); -} - -function seedLegacyProjection(source: Database, target: Database, threadId: string) { - seedThread(target, readThread(source, threadId)); - - const messages = source - .query( - ` - SELECT - message_id AS messageId, - thread_id AS threadId, - turn_id AS turnId, - role, - text, - attachments_json AS attachmentsJson, - is_streaming AS isStreaming, - created_at AS createdAt, - updated_at AS updatedAt - FROM projection_thread_messages - WHERE thread_id = ? - `, - ) - .all(threadId); - const insertMessage = target.query(` - INSERT INTO projection_thread_messages ( - message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at - ) - VALUES ( - $messageId, $threadId, $turnId, $role, $text, $attachmentsJson, $isStreaming, $createdAt, $updatedAt - ) - `); - for (const message of messages) { - insertMessage.run(message as unknown as SqlBindingRecord); - } - - const activities = source - .query( - ` - SELECT - activity_id AS activityId, - thread_id AS threadId, - turn_id AS turnId, - tone, - kind, - summary, - payload_json AS payloadJson, - sequence, - created_at AS createdAt - FROM projection_thread_activities - WHERE thread_id = ? - `, - ) - .all(threadId); - const insertActivity = target.query(` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) - VALUES ( - $activityId, $threadId, $turnId, $tone, $kind, $summary, $payloadJson, $sequence, $createdAt - ) - `); - for (const activity of activities) { - insertActivity.run(activity as unknown as SqlBindingRecord); - } - - const plans = source - .query( - ` - SELECT - plan_id AS planId, - thread_id AS threadId, - turn_id AS turnId, - plan_markdown AS planMarkdown, - implemented_at AS implementedAt, - implementation_thread_id AS implementationThreadId, - created_at AS createdAt, - updated_at AS updatedAt - FROM projection_thread_proposed_plans - WHERE thread_id = ? - `, - ) - .all(threadId); - const insertPlan = target.query(` - INSERT INTO projection_thread_proposed_plans ( - plan_id, thread_id, turn_id, plan_markdown, implemented_at, implementation_thread_id, created_at, updated_at - ) - VALUES ( - $planId, $threadId, $turnId, $planMarkdown, $implementedAt, $implementationThreadId, $createdAt, $updatedAt - ) - `); - for (const plan of plans) { - insertPlan.run(plan as unknown as SqlBindingRecord); - } - - const approvals = source - .query( - ` - SELECT - request_id AS approvalId, - thread_id AS threadId, - turn_id AS turnId, - status, - created_at AS createdAt, - COALESCE(resolved_at, created_at) AS updatedAt - FROM projection_pending_approvals - WHERE thread_id = ? - `, - ) - .all(threadId); - const insertApproval = target.query(` - INSERT INTO projection_pending_approvals ( - approval_id, thread_id, turn_id, status, created_at, updated_at - ) - VALUES ( - $approvalId, $threadId, $turnId, $status, $createdAt, $updatedAt - ) - `); - for (const approval of approvals) { - insertApproval.run(approval as unknown as SqlBindingRecord); - } -} - -function seedOptimizedProjection(source: Database, target: Database, threadId: string) { - seedThread(target, readThread(source, threadId)); -} - -function makeOptimizedRunner(db: Database) { - const append = db.query(` - INSERT INTO projection_thread_messages ( - message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at - ) - VALUES ( - $messageId, $threadId, $turnId, $role, $text, NULL, 1, $createdAt, $updatedAt - ) - ON CONFLICT (message_id) - DO UPDATE SET - thread_id = excluded.thread_id, - turn_id = excluded.turn_id, - role = excluded.role, - text = projection_thread_messages.text || excluded.text, - is_streaming = excluded.is_streaming, - updated_at = excluded.updated_at - `); - const touchThread = db.query(` - UPDATE projection_threads - SET updated_at = $updatedAt - WHERE thread_id = $threadId - `); - - return (event: AssistantStreamingEvent): number => { - const started = performance.now(); - append.run({ - messageId: event.messageId, - threadId: event.threadId, - turnId: event.turnId, - role: event.role, - text: event.text, - createdAt: event.createdAt, - updatedAt: event.updatedAt, - }); - touchThread.run({ threadId: event.threadId, updatedAt: event.updatedAt }); - return performance.now() - started; - }; -} - -function makeLegacyRunner(db: Database) { - const getExistingText = db.query<{ readonly text: string }, [string]>( - `SELECT text FROM projection_thread_messages WHERE message_id = ?`, - ); - const upsertFullText = db.query(` - INSERT INTO projection_thread_messages ( - message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, created_at, updated_at - ) - VALUES ( - $messageId, $threadId, $turnId, $role, $text, NULL, 1, $createdAt, $updatedAt - ) - ON CONFLICT (message_id) - DO UPDATE SET - thread_id = excluded.thread_id, - turn_id = excluded.turn_id, - role = excluded.role, - text = excluded.text, - is_streaming = excluded.is_streaming, - updated_at = excluded.updated_at - `); - const touchThread = db.query(` - UPDATE projection_threads - SET updated_at = $updatedAt - WHERE thread_id = $threadId - `); - const listMessages = db.query(` - SELECT message_id, role, text, attachments_json, is_streaming, created_at, updated_at - FROM projection_thread_messages - WHERE thread_id = ? - ORDER BY created_at ASC, message_id ASC - `); - const listActivities = db.query(` - SELECT activity_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - FROM projection_thread_activities - WHERE thread_id = ? - ORDER BY - CASE WHEN sequence IS NULL THEN 0 ELSE 1 END ASC, - sequence ASC, - created_at ASC, - activity_id ASC - `); - const listPlans = db.query(` - SELECT plan_id, turn_id, plan_markdown, implemented_at, implementation_thread_id, created_at, updated_at - FROM projection_thread_proposed_plans - WHERE thread_id = ? - ORDER BY created_at ASC, plan_id ASC - `); - const pendingApprovals = db.query(` - SELECT approval_id - FROM projection_pending_approvals - WHERE thread_id = ? AND status = 'pending' - `); - - return (event: AssistantStreamingEvent): number => { - const started = performance.now(); - db.exec("BEGIN"); - try { - const existing = getExistingText.get(event.messageId); - const nextText = `${existing?.text ?? ""}${event.text}`; - upsertFullText.run({ - messageId: event.messageId, - threadId: event.threadId, - turnId: event.turnId, - role: event.role, - text: nextText, - createdAt: event.createdAt, - updatedAt: event.updatedAt, - }); - touchThread.run({ threadId: event.threadId, updatedAt: event.updatedAt }); - - // This mirrors the old per-delta shell summary refresh shape: hydrate the - // thread's message/activity/plan/approval read-model state after every token. - const messages = listMessages.all(event.threadId) as ReadonlyArray<{ - readonly role: string; - readonly created_at: string; - }>; - const activities = listActivities.all(event.threadId) as ReadonlyArray<{ - readonly kind: string; - readonly payload_json: string; - }>; - const plans = listPlans.all(event.threadId); - const approvals = pendingApprovals.all(event.threadId); - const latestUserMessageAt = messages - .filter((message) => message.role === "user") - .map((message) => message.created_at) - .toSorted() - .at(-1); - const pendingUserInputCount = activities.filter((activity) => { - if (activity.kind !== "input") { - return false; - } - try { - const payload = JSON.parse(activity.payload_json) as { readonly state?: string }; - return payload.state === "pending"; - } catch { - return false; - } - }).length; - const derivedSummary = { - latestUserMessageAt, - pendingUserInputCount, - planCount: plans.length, - pendingApprovalCount: approvals.length, - }; - _summaryRefreshSink = JSON.stringify(derivedSummary); - - db.exec("ROLLBACK"); - return performance.now() - started; - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - }; -} - -function selectLegacyEvents( - events: ReadonlyArray, - options: CliOptions, -): ReadonlyArray { - if (options.legacyMode === "full" || events.length <= options.legacySamplePerWindow) { - return events; - } - - const selected: Array = []; - const step = events.length / options.legacySamplePerWindow; - for (let index = 0; index < options.legacySamplePerWindow; index += 1) { - const event = events[Math.min(events.length - 1, Math.floor(index * step))]; - if (event) { - selected.push(event); - } - } - return selected; -} - -function formatMs(value: number): string { - return value.toFixed(4); -} - -function divideForSpeedup(left: number, right: number): number { - return right === 0 ? 0 : left / right; -} - -function printCompareStats(input: { - readonly label: string; - readonly optimized: ReadonlyArray; - readonly legacy: ReadonlyArray; -}) { - const optimized = calculateTimingStats(input.optimized); - const legacy = calculateTimingStats(input.legacy); - console.log(input.label); - console.log( - ` optimized: count=${optimized.count} mean=${formatMs(optimized.meanMs)}ms p50=${formatMs( - optimized.p50Ms, - )}ms p90=${formatMs(optimized.p90Ms)}ms p99=${formatMs(optimized.p99Ms)}ms max=${formatMs( - optimized.maxMs, - )}ms`, - ); - console.log( - ` legacy: count=${legacy.count} mean=${formatMs(legacy.meanMs)}ms p50=${formatMs( - legacy.p50Ms, - )}ms p90=${formatMs(legacy.p90Ms)}ms p99=${formatMs(legacy.p99Ms)}ms max=${formatMs( - legacy.maxMs, - )}ms`, - ); - console.log( - ` speedup: mean=${formatMs(divideForSpeedup(legacy.meanMs, optimized.meanMs))}x p50=${formatMs( - divideForSpeedup(legacy.p50Ms, optimized.p50Ms), - )}x p90=${formatMs(divideForSpeedup(legacy.p90Ms, optimized.p90Ms))}x p99=${formatMs( - divideForSpeedup(legacy.p99Ms, optimized.p99Ms), - )}x`, - ); -} - -function runBenchmark(options: CliOptions) { - const source = openSourceDb(options.sourceDb); - const events = readAssistantStreamingEvents(source, options); - if (events.length === 0) { - throw new Error(`No assistant streaming events found for thread ${options.threadId}`); - } - - const optimizedDb = createCompareDb(); - const legacyDb = createCompareDb(); - seedOptimizedProjection(source, optimizedDb, options.threadId); - seedLegacyProjection(source, legacyDb, options.threadId); - - const runOptimized = makeOptimizedRunner(optimizedDb); - const runLegacy = makeLegacyRunner(legacyDb); - const optimizedTimings: Array = []; - const legacyTimings: Array = []; - const windows: Array<{ - readonly from: number; - readonly to: number; - readonly optimized: ReadonlyArray; - readonly legacy: ReadonlyArray; - }> = []; - - for (let start = 0; start < events.length; start += options.windowSize) { - const end = Math.min(events.length, start + options.windowSize); - const windowEvents = events.slice(start, end); - const windowOptimized: Array = []; - const windowLegacy: Array = []; - - for (const event of windowEvents) { - const timing = runOptimized(event); - windowOptimized.push(timing); - optimizedTimings.push(timing); - } - - for (const event of selectLegacyEvents(windowEvents, options)) { - const timing = runLegacy(event); - windowLegacy.push(timing); - legacyTimings.push(timing); - } - - windows.push({ - from: start + 1, - to: end, - optimized: windowOptimized, - legacy: windowLegacy, - }); - console.log( - `compare progress ${end}/${events.length}: optimized=${windowOptimized.length} legacy=${windowLegacy.length}`, - ); - } - - console.log(`thread=${options.threadId}`); - console.log(`sourceDb=${options.sourceDb}`); - console.log("mode=assistant-streaming"); - console.log(`legacyMode=${options.legacyMode}`); - console.log(`windowSize=${options.windowSize}`); - console.log(`assistantStreamingEvents=${events.length}`); - if (options.legacyMode === "sampled") { - console.log(`legacySamplePerWindow=${options.legacySamplePerWindow}`); - } - printCompareStats({ - label: "overall", - optimized: optimizedTimings, - legacy: legacyTimings, - }); - for (const window of windows) { - printCompareStats({ - label: `window ${window.from}-${window.to} optimizedEvents=${window.optimized.length} legacyEvents=${window.legacy.length}`, - optimized: window.optimized, - legacy: window.legacy, - }); - } -} - -try { - runBenchmark(parseArgs(process.argv.slice(2))); -} catch (error) { - console.error(error instanceof Error ? error.message : error); - process.exit(1); -} diff --git a/apps/server/scripts/replay_thread/core.test.ts b/apps/server/scripts/replay_thread/core.test.ts deleted file mode 100644 index f0ff1849db..0000000000 --- a/apps/server/scripts/replay_thread/core.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { assert, it } from "@effect/vitest"; - -import { buildTimingSamples, calculateTimingStats } from "./core.ts"; - -it("calculates timing stats with stable percentile boundaries", () => { - assert.deepEqual(calculateTimingStats([]), { - count: 0, - totalMs: 0, - meanMs: 0, - p50Ms: 0, - p90Ms: 0, - p99Ms: 0, - maxMs: 0, - }); - - assert.deepEqual(calculateTimingStats([4, 1, 2, 3]), { - count: 4, - totalMs: 10, - meanMs: 2.5, - p50Ms: 2, - p90Ms: 4, - p99Ms: 4, - maxMs: 4, - }); -}); - -it("builds contiguous timing samples", () => { - assert.deepEqual(buildTimingSamples([1, 2, 3, 4, 5], 2), [ - { - fromEvent: 1, - toEvent: 2, - stats: calculateTimingStats([1, 2]), - }, - { - fromEvent: 3, - toEvent: 4, - stats: calculateTimingStats([3, 4]), - }, - { - fromEvent: 5, - toEvent: 5, - stats: calculateTimingStats([5]), - }, - ]); -}); diff --git a/apps/server/scripts/replay_thread/core.ts b/apps/server/scripts/replay_thread/core.ts deleted file mode 100644 index ac96b833be..0000000000 --- a/apps/server/scripts/replay_thread/core.ts +++ /dev/null @@ -1,63 +0,0 @@ -export interface TimingStats { - readonly count: number; - readonly totalMs: number; - readonly meanMs: number; - readonly p50Ms: number; - readonly p90Ms: number; - readonly p99Ms: number; - readonly maxMs: number; -} - -export interface ReplayTimingSample { - readonly fromEvent: number; - readonly toEvent: number; - readonly stats: TimingStats; -} - -export function calculateTimingStats(values: ReadonlyArray): TimingStats { - if (values.length === 0) { - return { - count: 0, - totalMs: 0, - meanMs: 0, - p50Ms: 0, - p90Ms: 0, - p99Ms: 0, - maxMs: 0, - }; - } - - const sorted = [...values].toSorted((left, right) => left - right); - const totalMs = values.reduce((total, value) => total + value, 0); - const percentile = (percent: number) => { - const index = Math.min(sorted.length - 1, Math.ceil((percent / 100) * sorted.length) - 1); - return sorted[Math.max(0, index)] ?? 0; - }; - - return { - count: values.length, - totalMs, - meanMs: totalMs / values.length, - p50Ms: percentile(50), - p90Ms: percentile(90), - p99Ms: percentile(99), - maxMs: sorted[sorted.length - 1] ?? 0, - }; -} - -export function buildTimingSamples( - timings: ReadonlyArray, - sampleEvery: number, -): ReadonlyArray { - const normalizedSampleEvery = Math.max(1, Math.floor(sampleEvery)); - const samples: Array = []; - for (let start = 0; start < timings.length; start += normalizedSampleEvery) { - const end = Math.min(timings.length, start + normalizedSampleEvery); - samples.push({ - fromEvent: start + 1, - toEvent: end, - stats: calculateTimingStats(timings.slice(start, end)), - }); - } - return samples; -} From b28e5f2a4ce7e45f2c28e4d9a3c607db82139634 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Sun, 10 May 2026 18:48:09 +0100 Subject: [PATCH 07/10] Fix projection test timestamp construction --- .../orchestration/Layers/ProjectionPipeline.test.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 94028bf2b3..c74b33ea8d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -11,6 +11,7 @@ import { } from "@t3tools/contracts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -612,9 +613,9 @@ it.layer( const projectionPipeline = yield* OrchestrationProjectionPipeline; const eventStore = yield* OrchestrationEventStore; const sql = yield* SqlClient.SqlClient; - const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); - const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); - const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); + const first = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:01.000Z")); + const second = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:02.000Z")); + const third = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:03.000Z")); const appendAndProject = (event: Parameters[0]) => eventStore @@ -730,9 +731,9 @@ it.layer( const projectionPipeline = yield* OrchestrationProjectionPipeline; const snapshotQuery = yield* ProjectionSnapshotQuery; const eventStore = yield* OrchestrationEventStore; - const first = new Date("2026-02-24T00:00:01.000Z").toISOString(); - const second = new Date("2026-02-24T00:00:02.000Z").toISOString(); - const third = new Date("2026-02-24T00:00:03.000Z").toISOString(); + const first = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:01.000Z")); + const second = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:02.000Z")); + const third = DateTime.formatIso(DateTime.makeUnsafe("2026-02-24T00:00:03.000Z")); const appendAndProject = (event: Parameters[0]) => eventStore From bdeac91f54cab76f3fc859baa3b0c10c67d6a896 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Mon, 11 May 2026 10:06:54 +0100 Subject: [PATCH 08/10] Move message row encoding into schemas --- .../Layers/ProjectionThreadMessages.ts | 70 +++++++++++++------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts index 67396be6c0..2927148aa0 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts @@ -7,7 +7,7 @@ import * as Schema from "effect/Schema"; import * as Struct from "effect/Struct"; import { ChatAttachment } from "@t3tools/contracts"; -import { toPersistenceSqlError } from "../Errors.ts"; +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; import { AppendProjectionThreadMessageTextInput, GetProjectionThreadMessageInput, @@ -25,6 +25,24 @@ const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( }), ); +const ProjectionThreadMessageDbInput = ProjectionThreadMessage.mapFields( + Struct.assign({ + isStreaming: Schema.BooleanFromBit, + attachments: Schema.optional(Schema.fromJsonString(Schema.Array(ChatAttachment))), + }), +); + +const AppendProjectionThreadMessageTextDbInput = AppendProjectionThreadMessageTextInput.mapFields( + Struct.assign({ + isStreaming: Schema.BooleanFromBit, + attachments: Schema.optional(Schema.fromJsonString(Schema.Array(ChatAttachment))), + }), +); +const encodeProjectionThreadMessageDbInput = Schema.encodeEffect(ProjectionThreadMessageDbInput); +const encodeAppendProjectionThreadMessageTextDbInput = Schema.encodeEffect( + AppendProjectionThreadMessageTextDbInput, +); + function toProjectionThreadMessage( row: Schema.Schema.Type, ): ProjectionThreadMessage { @@ -45,11 +63,8 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const upsertProjectionThreadMessageRow = SqlSchema.void({ - Request: ProjectionThreadMessage, - execute: (row) => { - const nextAttachmentsJson = - row.attachments !== undefined ? JSON.stringify(row.attachments) : null; - return sql` + Request: Schema.toEncoded(ProjectionThreadMessageDbInput), + execute: (row) => sql` INSERT INTO projection_thread_messages ( message_id, thread_id, @@ -68,14 +83,14 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { ${row.role}, ${row.text}, COALESCE( - ${nextAttachmentsJson}, + ${row.attachments ?? null}, ( SELECT attachments_json FROM projection_thread_messages WHERE message_id = ${row.messageId} ) ), - ${row.isStreaming ? 1 : 0}, + ${row.isStreaming}, ${row.createdAt}, ${row.updatedAt} ) @@ -92,16 +107,12 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { is_streaming = excluded.is_streaming, created_at = excluded.created_at, updated_at = excluded.updated_at - `; - }, + `, }); const appendProjectionThreadMessageText = SqlSchema.void({ - Request: AppendProjectionThreadMessageTextInput, - execute: (row) => { - const nextAttachmentsJson = - row.attachments !== undefined ? JSON.stringify(row.attachments) : null; - return sql` + Request: Schema.toEncoded(AppendProjectionThreadMessageTextDbInput), + execute: (row) => sql` INSERT INTO projection_thread_messages ( message_id, thread_id, @@ -119,8 +130,8 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { ${row.turnId}, ${row.role}, ${row.textDelta}, - ${nextAttachmentsJson}, - ${row.isStreaming ? 1 : 0}, + ${row.attachments ?? null}, + ${row.isStreaming}, ${row.createdAt}, ${row.updatedAt} ) @@ -136,8 +147,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { ), is_streaming = excluded.is_streaming, updated_at = excluded.updated_at - `; - }, + `, }); const getProjectionThreadMessageRow = SqlSchema.findOneOption({ @@ -192,13 +202,27 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { }); const upsert: ProjectionThreadMessageRepositoryShape["upsert"] = (row) => - upsertProjectionThreadMessageRow(row).pipe( - Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.upsert:query")), + encodeProjectionThreadMessageDbInput(row).pipe( + Effect.mapError(toPersistenceDecodeError("ProjectionThreadMessageRepository.upsert:encode")), + Effect.flatMap((encodedRow) => + upsertProjectionThreadMessageRow(encodedRow).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.upsert:query")), + ), + ), ); const appendText: ProjectionThreadMessageRepositoryShape["appendText"] = (input) => - appendProjectionThreadMessageText(input).pipe( - Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.appendText:query")), + encodeAppendProjectionThreadMessageTextDbInput(input).pipe( + Effect.mapError( + toPersistenceDecodeError("ProjectionThreadMessageRepository.appendText:encode"), + ), + Effect.flatMap((encodedInput) => + appendProjectionThreadMessageText(encodedInput).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadMessageRepository.appendText:query"), + ), + ), + ), ); const getByMessageId: ProjectionThreadMessageRepositoryShape["getByMessageId"] = (input) => From 482b98638f3ded792ae3284f0cfdbf64e4bef469 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Mon, 11 May 2026 18:39:29 +0100 Subject: [PATCH 09/10] Provide node services to projection snapshot test --- apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index c74b33ea8d..e7f717618d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -604,7 +604,7 @@ it.layer( Layer.provideMerge( makeProjectionPipelinePrefixedTestLayer("t3-projection-streaming-append-"), ), - Layer.provideMerge(RepositoryIdentityResolverLive), + Layer.provideMerge(RepositoryIdentityResolverLive.pipe(Layer.provide(NodeServices.layer))), ), ), )("OrchestrationProjectionPipeline", (it) => { From 2ae1ae1cfcffecee552e8f761591cf0620f05fe1 Mon Sep 17 00:00:00 2001 From: justsomelegs <145564979+justsomelegs@users.noreply.github.com> Date: Mon, 11 May 2026 18:49:33 +0100 Subject: [PATCH 10/10] Align projection snapshot test layer wiring --- .../server/src/orchestration/Layers/ProjectionPipeline.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index e7f717618d..cb93b8d6ab 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -604,7 +604,8 @@ it.layer( Layer.provideMerge( makeProjectionPipelinePrefixedTestLayer("t3-projection-streaming-append-"), ), - Layer.provideMerge(RepositoryIdentityResolverLive.pipe(Layer.provide(NodeServices.layer))), + Layer.provideMerge(RepositoryIdentityResolverLive), + Layer.provideMerge(NodeServices.layer), ), ), )("OrchestrationProjectionPipeline", (it) => {