From 71a257a6d35c160317d4545c72423befd6c3951c Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 19:08:11 +0100 Subject: [PATCH 1/5] fix(api): use internal user ID instead of WorkOS ID for conversation operations All conversation API routes now look up the internal database user ID from the WorkOS ID before performing operations. This ensures consistency with the database schema where conversations are linked to internal user IDs. Also relaxed message schema validation to support all AI SDK part types (text, image, tool-call, tool-result, reasoning, file, etc.) and added passthrough to prevent sync failures with legitimate messages. --- src/app/api/conversations/[id]/route.ts | 42 ++++++++++++++++++++++--- src/app/api/conversations/route.ts | 23 ++++++++++++-- src/features/john-gpt/schema.ts | 14 ++++----- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/app/api/conversations/[id]/route.ts b/src/app/api/conversations/[id]/route.ts index 60bb6ef..4fbd277 100644 --- a/src/app/api/conversations/[id]/route.ts +++ b/src/app/api/conversations/[id]/route.ts @@ -3,6 +3,7 @@ import { ConversationService } from '@/features/john-gpt/services/conversation.s import { UpdateConversationSchema } from '@/features/john-gpt/schema'; import { z } from 'zod'; import { withAuth } from '@workos-inc/authkit-nextjs'; +import { prisma } from '@/lib/prisma'; export async function GET( req: NextRequest, @@ -16,8 +17,18 @@ export async function GET( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + // Look up internal user ID (matching POST route pattern) + const internalUser = await prisma.user.findUnique({ + where: { workosId: user.id }, + select: { id: true }, + }); + + if (!internalUser) { + return NextResponse.json({ error: 'User not found' }, { status: 404 }); + } + try { - const conversation = await ConversationService.getConversation(id, user.id); + const conversation = await ConversationService.getConversation(id, internalUser.id); if (!conversation) { return NextResponse.json({ error: 'Not Found' }, { status: 404 }); @@ -42,17 +53,30 @@ export async function PATCH( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + // Look up internal user ID (matching POST route pattern) + const internalUser = await prisma.user.findUnique({ + where: { workosId: user.id }, + select: { id: true }, + }); + + if (!internalUser) { + return NextResponse.json({ error: 'User not found' }, { status: 404 }); + } + try { const body = await req.json(); const data = UpdateConversationSchema.parse(body); - const conversation = await ConversationService.updateConversation(id, user.id, data); + const conversation = await ConversationService.updateConversation(id, internalUser.id, data); return NextResponse.json(conversation); } catch (error) { if (error instanceof z.ZodError) { return NextResponse.json({ error: 'Validation Error', details: (error as any).errors || (error as any).issues }, { status: 400 }); } - // Handle specific Prisma errors like "Record not found" if needed + // Handle Prisma "Record not found" error - return 404 to trigger POST fallback + if ((error as any)?.code === 'P2025') { + return NextResponse.json({ error: 'Not Found' }, { status: 404 }); + } console.error('Failed to update conversation:', error); return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 }); } @@ -70,8 +94,18 @@ export async function DELETE( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + // Look up internal user ID (matching POST route pattern) + const internalUser = await prisma.user.findUnique({ + where: { workosId: user.id }, + select: { id: true }, + }); + + if (!internalUser) { + return NextResponse.json({ error: 'User not found' }, { status: 404 }); + } + try { - await ConversationService.deleteConversation(id, user.id); + await ConversationService.deleteConversation(id, internalUser.id); return NextResponse.json({ success: true }); } catch (error) { console.error('Failed to delete conversation:', error); diff --git a/src/app/api/conversations/route.ts b/src/app/api/conversations/route.ts index ae1a560..17bd269 100644 --- a/src/app/api/conversations/route.ts +++ b/src/app/api/conversations/route.ts @@ -3,6 +3,7 @@ import { ConversationService } from '@/features/john-gpt/services/conversation.s import { CreateConversationSchema } from '@/features/john-gpt/schema'; import { z } from 'zod'; import { withAuth } from '@workos-inc/authkit-nextjs'; +import { prisma } from '@/lib/prisma'; export async function GET(req: NextRequest) { const { user } = await withAuth(); @@ -11,8 +12,17 @@ export async function GET(req: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + const internalUser = await prisma.user.findUnique({ + where: { workosId: user.id }, + select: { id: true }, + }); + + if (!internalUser) { + return NextResponse.json({ error: 'User not found' }, { status: 404 }); + } + try { - const conversations = await ConversationService.listConversations(user.id); + const conversations = await ConversationService.listConversations(internalUser.id); return NextResponse.json(conversations); } catch (error) { console.error('Failed to list conversations:', error); @@ -27,12 +37,21 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + const internalUser = await prisma.user.findUnique({ + where: { workosId: user.id }, + select: { id: true }, + }); + + if (!internalUser) { + return NextResponse.json({ error: 'User not found' }, { status: 404 }); + } + try { const body = await req.json(); // Allow ID to be passed in body for client-side generation const data = CreateConversationSchema.extend({ id: z.string().optional() }).parse(body); - const conversation = await ConversationService.createConversation(user.id, data); + const conversation = await ConversationService.createConversation(internalUser.id, data); return NextResponse.json(conversation); } catch (error) { if (error instanceof z.ZodError) { diff --git a/src/features/john-gpt/schema.ts b/src/features/john-gpt/schema.ts index bbf1744..a00afd0 100644 --- a/src/features/john-gpt/schema.ts +++ b/src/features/john-gpt/schema.ts @@ -1,24 +1,24 @@ import { z } from 'zod'; // AI SDK Message format validation +// NOTE: Parts are kept permissive because the AI SDK uses many part types +// (text, image, tool-call, tool-result, reasoning, file, etc.) +// Strict validation would break sync for legitimate messages. export const MessagePartSchema = z.object({ - type: z.enum(['text', 'image', 'tool-invocation']), - text: z.string().optional(), - image: z.string().optional(), - toolInvocation: z.any().optional(), -}); + type: z.string(), // Permissive - AI SDK has many part types +}).passthrough(); // Allow any additional properties export const MessageSchema = z.object({ id: z.string(), role: z.enum(['system', 'user', 'assistant', 'data', 'tool']), - content: z.string().optional(), // Simple string content + content: z.string().optional().nullable(), // Simple string content (can be null) parts: z.array(MessagePartSchema).optional(), // Structured content createdAt: z.union([z.string(), z.date()]).optional(), metadata: z.record(z.string(), z.any()).optional(), // Branching Logic parentId: z.string().nullable().optional(), childrenIds: z.array(z.string()).optional(), -}); +}).passthrough(); // Allow additional AI SDK fields we don't explicitly define // Conversation CRUD schemas export const CreateConversationSchema = z.object({ From 39706f648c8899785fed339a7632c416c2ad3e8c Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 22:08:45 +0100 Subject: [PATCH 2/5] refactor(auth): add reusable auth helper and harden message schema validation - Created `src/lib/user-auth.ts` with `getAuthenticatedUser()` to eliminate N+1 user lookup patterns - Added standard response helpers: `unauthorizedResponse()`, `forbiddenResponse()`, `notFoundResponse()` - Refactored `/api/conversations/[id]/route.ts` to use shared auth helper - Added explicit 403 responses for ownership violations on PATCH/DELETE operations - Hardened `MessagePartSchema` with discriminated union of known AI SDK part types (replaced `.passthrough()`) - Updated documentation with auth helper reference and message schema validation details --- docs/features/WorkOSAuthentication.md | 47 ++++++++++ .../john-gpt/StorageAndPersistence.md | 32 ++++++- src/app/api/conversations/[id]/route.ts | 79 +++++++--------- src/features/john-gpt/schema.ts | 89 +++++++++++++++++-- src/lib/user-auth.ts | 79 ++++++++++++++++ 5 files changed, 271 insertions(+), 55 deletions(-) create mode 100644 src/lib/user-auth.ts diff --git a/docs/features/WorkOSAuthentication.md b/docs/features/WorkOSAuthentication.md index b65700d..d2d8bbf 100644 --- a/docs/features/WorkOSAuthentication.md +++ b/docs/features/WorkOSAuthentication.md @@ -360,3 +360,50 @@ export default authkitMiddleware({ - Verified all tables created successfully in Supabase - Tested WorkOS authentication with PostgreSQL user sync - See [`DATABASE_INTEGRATION.md`](./DATABASE_INTEGRATION.md) for complete database documentation + +### v2.1.0 - API Auth Helper & Security Hardening (2024-12-14) +- **Created `src/lib/user-auth.ts`** - Reusable auth helper to eliminate N+1 queries +- Added `getAuthenticatedUser()` for combined WorkOS + DB user lookup +- Added `unauthorizedResponse()`, `forbiddenResponse()`, `notFoundResponse()` helpers +- Refactored `/api/conversations/[id]/route.ts` to use shared helper +- Added explicit 403 responses for ownership violations (PATCH/DELETE) + +--- + +## 14. Auth Helper Reference + +### `user-auth.ts` (`src/lib/user-auth.ts`) + +**Purpose**: Eliminates N+1 user lookup patterns by combining WorkOS authentication and internal database user resolution in a single reusable helper. + +#### Exports + +| Export | Type | Description | +|--------|------|-------------| +| `getAuthenticatedUser()` | `async function` | Returns `AuthResult` or `null` | +| `unauthorizedResponse()` | `function` | 401 response | +| `forbiddenResponse(msg?)` | `function` | 403 response | +| `notFoundResponse(resource?)` | `function` | 404 response | + +#### Usage Example + +```typescript +import { getAuthenticatedUser, unauthorizedResponse, forbiddenResponse } from '@/lib/user-auth'; + +export async function GET(req: NextRequest) { + const authResult = await getAuthenticatedUser(); + + if (!authResult) { + return unauthorizedResponse(); + } + + const { internalUser } = authResult; + const data = await SomeService.getData(internalUser.id); + + if (!data) { + return forbiddenResponse('Access denied'); + } + + return NextResponse.json(data); +} +``` diff --git a/docs/features/john-gpt/StorageAndPersistence.md b/docs/features/john-gpt/StorageAndPersistence.md index 9001e5f..d3f0654 100644 --- a/docs/features/john-gpt/StorageAndPersistence.md +++ b/docs/features/john-gpt/StorageAndPersistence.md @@ -104,8 +104,38 @@ const conversation = await syncManager.loadConversation(conversationId); await syncManager.initializeGoogleDrive(userId); ``` -## 7. Future Improvements +## 7. Message Schema Validation + +The `MessagePartSchema` (`src/features/john-gpt/schema.ts`) validates AI SDK message parts before storage. + +### Supported Part Types + +| Type | Fields | +|------|--------| +| `text` | `text: string` | +| `image` | `image: string`, `mimeType?: string` | +| `file` | `data: string`, `mimeType: string` | +| `tool-call` | `toolCallId`, `toolName`, `args` | +| `tool-result` | `toolCallId`, `toolName`, `result`, `isError?` | +| `reasoning` | `reasoning: string` | +| `source` | `source: { sourceType, id, url?, title? }` | +| `step-start/finish` | *(no additional fields)* | + +> **Note:** If new AI SDK part types are added, update the discriminated union in `schema.ts`. + +--- + +## 8. Future Improvements * **Conflict Resolution UI:** Allow users to choose versions if a conflict occurs. * **Storage Quota:** Display Drive usage. * **Export:** Download conversation as Markdown/PDF. + +--- + +## 9. Change Log + +| Date | Change | +|------|--------| +| 2024-12-14 | Hardened `MessagePartSchema` with explicit discriminated union (was `.passthrough()`) | +| 2024-12-14 | Added ownership checks (403 responses) to `/api/conversations/[id]` | diff --git a/src/app/api/conversations/[id]/route.ts b/src/app/api/conversations/[id]/route.ts index 4fbd277..725beab 100644 --- a/src/app/api/conversations/[id]/route.ts +++ b/src/app/api/conversations/[id]/route.ts @@ -2,36 +2,31 @@ import { NextRequest, NextResponse } from 'next/server'; import { ConversationService } from '@/features/john-gpt/services/conversation.service'; import { UpdateConversationSchema } from '@/features/john-gpt/schema'; import { z } from 'zod'; -import { withAuth } from '@workos-inc/authkit-nextjs'; -import { prisma } from '@/lib/prisma'; +import { + getAuthenticatedUser, + unauthorizedResponse, + notFoundResponse, + forbiddenResponse +} from '@/lib/user-auth'; export async function GET( req: NextRequest, props: { params: Promise<{ id: string }> } ) { - const { params } = props; - const { user } = await withAuth(); - const { id } = await params; + const authResult = await getAuthenticatedUser(); + const { id } = await props.params; - if (!user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + if (!authResult) { + return unauthorizedResponse(); } - // Look up internal user ID (matching POST route pattern) - const internalUser = await prisma.user.findUnique({ - where: { workosId: user.id }, - select: { id: true }, - }); - - if (!internalUser) { - return NextResponse.json({ error: 'User not found' }, { status: 404 }); - } + const { internalUser } = authResult; try { const conversation = await ConversationService.getConversation(id, internalUser.id); if (!conversation) { - return NextResponse.json({ error: 'Not Found' }, { status: 404 }); + return notFoundResponse('Conversation'); } return NextResponse.json(conversation); @@ -45,23 +40,14 @@ export async function PATCH( req: NextRequest, props: { params: Promise<{ id: string }> } ) { - const { params } = props; - const { user } = await withAuth(); - const { id } = await params; + const authResult = await getAuthenticatedUser(); + const { id } = await props.params; - if (!user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + if (!authResult) { + return unauthorizedResponse(); } - // Look up internal user ID (matching POST route pattern) - const internalUser = await prisma.user.findUnique({ - where: { workosId: user.id }, - select: { id: true }, - }); - - if (!internalUser) { - return NextResponse.json({ error: 'User not found' }, { status: 404 }); - } + const { internalUser } = authResult; try { const body = await req.json(); @@ -73,9 +59,11 @@ export async function PATCH( if (error instanceof z.ZodError) { return NextResponse.json({ error: 'Validation Error', details: (error as any).errors || (error as any).issues }, { status: 400 }); } - // Handle Prisma "Record not found" error - return 404 to trigger POST fallback + // Handle Prisma "Record not found" error - could be 404 (not exists) or 403 (wrong owner) + // Since our WHERE clause includes userId, P2025 means either the conversation doesn't exist + // or it belongs to a different user. We return 403 to be safe (assumes ID is valid format). if ((error as any)?.code === 'P2025') { - return NextResponse.json({ error: 'Not Found' }, { status: 404 }); + return forbiddenResponse('Conversation not found or access denied'); } console.error('Failed to update conversation:', error); return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 }); @@ -86,28 +74,25 @@ export async function DELETE( req: NextRequest, props: { params: Promise<{ id: string }> } ) { - const { params } = props; - const { user } = await withAuth(); - const { id } = await params; + const authResult = await getAuthenticatedUser(); + const { id } = await props.params; - if (!user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + if (!authResult) { + return unauthorizedResponse(); } - // Look up internal user ID (matching POST route pattern) - const internalUser = await prisma.user.findUnique({ - where: { workosId: user.id }, - select: { id: true }, - }); - - if (!internalUser) { - return NextResponse.json({ error: 'User not found' }, { status: 404 }); - } + const { internalUser } = authResult; try { await ConversationService.deleteConversation(id, internalUser.id); return NextResponse.json({ success: true }); } catch (error) { + // Handle Prisma "Record not found" error - could be 404 or 403 + // Since our WHERE clause includes userId, P2025 means either the conversation doesn't exist + // or it belongs to a different user. We return 403 to be safe. + if ((error as any)?.code === 'P2025') { + return forbiddenResponse('Conversation not found or access denied'); + } console.error('Failed to delete conversation:', error); return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 }); } diff --git a/src/features/john-gpt/schema.ts b/src/features/john-gpt/schema.ts index a00afd0..8f0f9db 100644 --- a/src/features/john-gpt/schema.ts +++ b/src/features/john-gpt/schema.ts @@ -1,12 +1,87 @@ import { z } from 'zod'; -// AI SDK Message format validation -// NOTE: Parts are kept permissive because the AI SDK uses many part types -// (text, image, tool-call, tool-result, reasoning, file, etc.) -// Strict validation would break sync for legitimate messages. -export const MessagePartSchema = z.object({ - type: z.string(), // Permissive - AI SDK has many part types -}).passthrough(); // Allow any additional properties +// ============================================================================ +// AI SDK Message Part Schemas +// ============================================================================ +// These schemas define the known safe part types from the AI SDK. +// We use a discriminated union to validate structure while remaining +// type-safe against arbitrary/malicious payloads. +// Reference: https://sdk.vercel.ai/docs/ai-sdk-ui/chatbot-message-types + +/** Text content part */ +const TextPartSchema = z.object({ + type: z.literal('text'), + text: z.string(), +}); + +/** Image content part (base64 or URL) */ +const ImagePartSchema = z.object({ + type: z.literal('image'), + image: z.string(), // Base64 data or URL + mimeType: z.string().optional(), +}); + +/** File reference part */ +const FilePartSchema = z.object({ + type: z.literal('file'), + data: z.string(), + mimeType: z.string(), +}); + +/** Tool invocation part (when assistant calls a tool) */ +const ToolCallPartSchema = z.object({ + type: z.literal('tool-call'), + toolCallId: z.string(), + toolName: z.string(), + args: z.record(z.string(), z.any()), +}); + +/** Tool result part (response from tool execution) */ +const ToolResultPartSchema = z.object({ + type: z.literal('tool-result'), + toolCallId: z.string(), + toolName: z.string(), + result: z.any(), + isError: z.boolean().optional(), +}); + +/** Reasoning/thinking part (for models that expose reasoning) */ +const ReasoningPartSchema = z.object({ + type: z.literal('reasoning'), + reasoning: z.string(), +}); + +/** Source reference part (for RAG/citations) */ +const SourcePartSchema = z.object({ + type: z.literal('source'), + source: z.object({ + sourceType: z.string(), + id: z.string(), + url: z.string().optional(), + title: z.string().optional(), + }), +}); + +/** Step start/finish markers */ +const StepPartSchema = z.object({ + type: z.union([z.literal('step-start'), z.literal('step-finish')]), +}); + +/** + * Discriminated union of all known AI SDK message part types. + * Unknown part types will fail validation, protecting against malformed + * or malicious data being stored in the database. + */ +export const MessagePartSchema = z.discriminatedUnion('type', [ + TextPartSchema, + ImagePartSchema, + FilePartSchema, + ToolCallPartSchema, + ToolResultPartSchema, + ReasoningPartSchema, + SourcePartSchema, + StepPartSchema, +]); export const MessageSchema = z.object({ id: z.string(), diff --git a/src/lib/user-auth.ts b/src/lib/user-auth.ts new file mode 100644 index 0000000..132ef58 --- /dev/null +++ b/src/lib/user-auth.ts @@ -0,0 +1,79 @@ +import { withAuth } from '@workos-inc/authkit-nextjs'; +import { NextResponse } from 'next/server'; +import { prisma } from '@/lib/prisma'; + +/** + * Internal user representation from database + */ +export interface InternalUser { + id: string; + workosId: string | null; +} + +/** + * Result of authentication with internal user lookup + */ +export interface AuthResult { + internalUser: InternalUser; + workosUser: NonNullable>['user']>; +} + +/** + * Authenticate and resolve the internal database user from WorkOS session. + * This helper eliminates N+1 query patterns by providing a single reusable + * lookup function for API routes. + * + * @returns AuthResult with both WorkOS user and internal database user, or null if unauthenticated + * @throws Never throws - returns null for unauthenticated requests + * + * @example + * ```ts + * const authResult = await getAuthenticatedUser(); + * if (!authResult) { + * return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + * } + * const { internalUser, workosUser } = authResult; + * ``` + */ +export async function getAuthenticatedUser(): Promise { + const { user: workosUser } = await withAuth(); + + if (!workosUser) { + return null; + } + + const internalUser = await prisma.user.findUnique({ + where: { workosId: workosUser.id }, + select: { id: true, workosId: true }, + }); + + if (!internalUser) { + return null; + } + + return { + internalUser, + workosUser, + }; +} + +/** + * Standard 401 Unauthorized response + */ +export function unauthorizedResponse(): NextResponse { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); +} + +/** + * Standard 403 Forbidden response for ownership violations + */ +export function forbiddenResponse(message = 'Access denied'): NextResponse { + return NextResponse.json({ error: message }, { status: 403 }); +} + +/** + * Standard 404 Not Found response + */ +export function notFoundResponse(resource = 'Resource'): NextResponse { + return NextResponse.json({ error: `${resource} not found` }, { status: 404 }); +} From 52d28bf4bcb9d2fd9a10ea70d8cb929a5cd02ddc Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Mon, 15 Dec 2025 09:52:35 +0100 Subject: [PATCH 3/5] fix(db): ensure cached conversations with messages are returned before API fetch Previously, loadConversation would return empty cached metadata from list endpoint and skip API fetch. Now validates cache has messages before returning, and fetches from API if cache is empty or metadata-only. Also improves listConversations to await API fetch on empty cache (fresh window scenario) instead of returning empty list, and adds comprehensive debug logging throughout sync operations. --- src/lib/storage/db-sync-manager.ts | 66 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/lib/storage/db-sync-manager.ts b/src/lib/storage/db-sync-manager.ts index bef9696..fe92aec 100644 --- a/src/lib/storage/db-sync-manager.ts +++ b/src/lib/storage/db-sync-manager.ts @@ -218,11 +218,15 @@ export class DBSyncManager { * Implements "stale-while-revalidate" pattern */ async loadConversation(conversationId: string, options?: { isWidget?: boolean }): Promise { + console.log(`[DBSyncManager] loadConversation called: ${conversationId}, isAuth: ${this.isAuthenticated}, isOnline: ${this.isOnline}`); + // 1. Try IndexedDB first (instant) const cached = await indexedDBClient.getConversation(conversationId); + const cachedMessageCount = cached?.messages?.length || 0; + console.log(`[DBSyncManager] IndexedDB cache result:`, cached ? `Found (${cachedMessageCount} msgs)` : 'Not found'); - if (cached) { - // Return cached data immediately + // Return cached data ONLY if it has messages (not just metadata from list endpoint) + if (cached && cachedMessageCount > 0) { // Background: fetch from API and update cache if newer if (this.isAuthenticated && !options?.isWidget) { this.revalidateFromApi(conversationId, cached.updatedAt).catch((error) => { @@ -232,17 +236,14 @@ export class DBSyncManager { return cached; } - // 2. No cache - fetch from API - // Widget sessions shouldn't be pulled from API? - // If user is authenticated, widget history IS chat history? - // Usually widget has separate session unless "Open in JohnGPT". - // If `isWidget` is true, we might want to skip API if API doesn't store widget sessions. - // Assuming API stores everything if `userId` matches. + // 2. No cache OR cached has no messages (metadata-only from list) - fetch from API if (!this.isOnline || !this.isAuthenticated) { + console.log(`[DBSyncManager] Skipping API fetch: online=${this.isOnline}, auth=${this.isAuthenticated}`); return null; // Can't fetch offline or if guest } try { + console.log(`[DBSyncManager] Fetching from API: /api/conversations/${conversationId}`); const res = await fetch(`/api/conversations/${conversationId}`); if (!res.ok) { if (res.status === 404) return null; @@ -250,11 +251,13 @@ export class DBSyncManager { } const conversation = await res.json(); + console.log(`[DBSyncManager] API returned conversation with ${conversation.messages?.length || 0} messages`); // Transform API response to CachedConversation const mapped = this.mapApiToCache(conversation); await this.updateCache(mapped, false); + console.log(`[DBSyncManager] Conversation cached and returning:`, mapped.messages?.length || 0, 'messages'); return mapped; } catch (error) { console.error('[DBSyncManager] Failed to load from API:', error); @@ -264,12 +267,22 @@ export class DBSyncManager { /** * List all conversations (from cache + API) + * If cache is empty and authenticated, fetches from API first (fresh window scenario) */ async listConversations(userId: string): Promise { // 1. Get from cache (instant) const cached = await indexedDBClient.listConversations(userId); - // 2. Background: refresh from API + // 2. If cache is empty and we're online+authenticated, await API fetch first + // This handles the "new browser window" scenario where IndexedDB is empty + if (cached.length === 0 && this.isOnline && this.isAuthenticated) { + console.log('[DBSyncManager] Cache empty, fetching from API...'); + await this.refreshConversationList(userId); + // Return the now-populated cache + return await indexedDBClient.listConversations(userId); + } + + // 3. Background refresh for non-empty cache (stale-while-revalidate) if (this.isOnline && this.isAuthenticated) { this.refreshConversationList(userId).catch((error) => { console.warn('[DBSyncManager] Background refresh failed:', error); @@ -482,46 +495,33 @@ export class DBSyncManager { private async refreshConversationList(userId: string): Promise { try { + console.log('[DBSyncManager] refreshConversationList: Fetching from API...'); const res = await fetch('/api/conversations'); - if (!res.ok) return; - // The API returns { conversations: [...] } or just [...] depending on implementation. - // Based on sidebar code: { conversations: [...] } - // Wait, previous Sidebar code used `data.conversations`. - // Let's verify API route return type. - // API route returns `NextResponse.json(conversations)` which is an array. - // Wait, Sidebar code `const data = await res.json(); const transformed = data.conversations...` - // Let's check api/conversations/route.ts again. - // It returns `NextResponse.json(conversations)`. - // So it IS an array directly. - // Sidebar code I replaced was `const data = await res.json(); // ... data.conversations.map ...` - // Wait, if API returns array, `data.conversations` would be undefined. - // Let me check my memory or previous file read of route.ts. - // route.ts: `return NextResponse.json(conversations);` -> Array. - // Sidebar original code: `const data = await res.json(); setConversations(data.conversations...` - // This suggests the Sidebar code MIGHT HAVE BEEN BROKEN or I misread route.ts. - // Let's assume API returns Array based on route.ts code `const conversations = ... findMany ... return json(conversations)`. + if (!res.ok) { + console.warn('[DBSyncManager] refreshConversationList: API returned', res.status); + return; + } const serverConversations = await res.json(); - - let hasChanges = false; + console.log(`[DBSyncManager] refreshConversationList: API returned ${Array.isArray(serverConversations) ? serverConversations.length : 0} conversations`); if (Array.isArray(serverConversations)) { for (const serverConv of serverConversations) { const cached = await indexedDBClient.getConversation(serverConv.id); const serverTime = new Date(serverConv.updatedAt).getTime(); - // If not in cache, or server is newer + // If not in cache, or server is newer - save to cache if (!cached || serverTime > new Date(cached.updatedAt).getTime()) { const mapped = this.mapApiToCache(serverConv); await this.updateCache(mapped, false); - hasChanges = true; + console.log(`[DBSyncManager] Cached conversation: ${serverConv.id} (${serverConv.title})`); } } } - if (hasChanges) { - this.notifyListListeners(); - } + // Always notify after refresh so UI updates + console.log('[DBSyncManager] refreshConversationList: Complete, notifying listeners'); + this.notifyListListeners(); } catch (e) { console.warn('[DBSyncManager] Background refresh failed:', e); } From ee2372d2b12f7d40c0c82568389dfe309788b6db Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Mon, 15 Dec 2025 10:42:07 +0100 Subject: [PATCH 4/5] fix(sync): prevent infinite refresh loop and improve error handling - Add isRefreshingList flag to prevent concurrent/recursive refreshes - Only notify listeners when actual changes are detected - Fix userId mapping for API responses that don't include userId - Handle null values in optional fields (personaId, selectedModelId) - Add detailed logging for sync payload debugging - Improve Zod validation error logging with formatted output - Ensure messages array defaults to empty array if undefined --- src/app/api/conversations/[id]/route.ts | 3 +- src/lib/storage/db-sync-manager.ts | 50 ++++++++++++++++++++----- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/app/api/conversations/[id]/route.ts b/src/app/api/conversations/[id]/route.ts index 725beab..6e60094 100644 --- a/src/app/api/conversations/[id]/route.ts +++ b/src/app/api/conversations/[id]/route.ts @@ -57,7 +57,8 @@ export async function PATCH( return NextResponse.json(conversation); } catch (error) { if (error instanceof z.ZodError) { - return NextResponse.json({ error: 'Validation Error', details: (error as any).errors || (error as any).issues }, { status: 400 }); + console.error('[API] Zod validation error:', JSON.stringify(error.issues, null, 2)); + return NextResponse.json({ error: 'Validation Error', details: error.issues }, { status: 400 }); } // Handle Prisma "Record not found" error - could be 404 (not exists) or 403 (wrong owner) // Since our WHERE clause includes userId, P2025 means either the conversation doesn't exist diff --git a/src/lib/storage/db-sync-manager.ts b/src/lib/storage/db-sync-manager.ts index fe92aec..8b60b64 100644 --- a/src/lib/storage/db-sync-manager.ts +++ b/src/lib/storage/db-sync-manager.ts @@ -60,6 +60,7 @@ export class DBSyncManager { private isAuthenticated: boolean = false; private userId: string | null = null; private isDriveConnected: boolean = false; + private isRefreshingList: boolean = false; constructor() { if (typeof window !== 'undefined') { @@ -283,7 +284,8 @@ export class DBSyncManager { } // 3. Background refresh for non-empty cache (stale-while-revalidate) - if (this.isOnline && this.isAuthenticated) { + // Only if not already refreshing (prevents infinite loop) + if (this.isOnline && this.isAuthenticated && !this.isRefreshingList) { this.refreshConversationList(userId).catch((error) => { console.warn('[DBSyncManager] Background refresh failed:', error); }); @@ -322,14 +324,15 @@ export class DBSyncManager { // Private Methods - Helpers // ========================================================================== - private mapApiToCache(apiConv: any): CachedConversation { + private mapApiToCache(apiConv: any, overrideUserId?: string): CachedConversation { return { conversationId: apiConv.id, - userId: apiConv.userId, + // API list endpoint doesn't return userId, so use override or fallback to manager's userId + userId: overrideUserId || apiConv.userId || this.userId, title: apiConv.title, createdAt: apiConv.createdAt, updatedAt: apiConv.updatedAt, - messages: apiConv.messages as any[], + messages: apiConv.messages as any[] || [], lastSyncedAt: Date.now(), // fresh from API isDirty: 0, localVersion: apiConv.localVersion, @@ -439,11 +442,21 @@ export class DBSyncManager { const payload = { title: cached.title, messages: cleanMessages, - personaId: (cached as any).personaId, - selectedModelId: (cached as any).selectedModelId, + // Zod .optional() accepts undefined but NOT null + personaId: (cached as any).personaId ?? undefined, + selectedModelId: (cached as any).selectedModelId ?? undefined, localVersion: Number((cached as any).localVersion) || 1, // Force number }; + console.log('[DBSyncManager] Sync payload:', { + conversationId, + title: payload.title, + messageCount: payload.messages.length, + localVersion: payload.localVersion, + personaId: payload.personaId, + selectedModelId: payload.selectedModelId, + }); + let res = await fetch(`/api/conversations/${conversationId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, @@ -494,6 +507,14 @@ export class DBSyncManager { } private async refreshConversationList(userId: string): Promise { + // Prevent concurrent/recursive refreshes + if (this.isRefreshingList) { + console.log('[DBSyncManager] refreshConversationList: Already refreshing, skipping'); + return; + } + + this.isRefreshingList = true; + try { console.log('[DBSyncManager] refreshConversationList: Fetching from API...'); const res = await fetch('/api/conversations'); @@ -505,6 +526,8 @@ export class DBSyncManager { const serverConversations = await res.json(); console.log(`[DBSyncManager] refreshConversationList: API returned ${Array.isArray(serverConversations) ? serverConversations.length : 0} conversations`); + let hasChanges = false; + if (Array.isArray(serverConversations)) { for (const serverConv of serverConversations) { const cached = await indexedDBClient.getConversation(serverConv.id); @@ -512,18 +535,25 @@ export class DBSyncManager { // If not in cache, or server is newer - save to cache if (!cached || serverTime > new Date(cached.updatedAt).getTime()) { - const mapped = this.mapApiToCache(serverConv); + const mapped = this.mapApiToCache(serverConv, userId); await this.updateCache(mapped, false); + hasChanges = true; console.log(`[DBSyncManager] Cached conversation: ${serverConv.id} (${serverConv.title})`); } } } - // Always notify after refresh so UI updates - console.log('[DBSyncManager] refreshConversationList: Complete, notifying listeners'); - this.notifyListListeners(); + // Only notify if there were actual changes (prevents infinite loop) + if (hasChanges) { + console.log('[DBSyncManager] refreshConversationList: Changes found, notifying listeners'); + this.notifyListListeners(); + } else { + console.log('[DBSyncManager] refreshConversationList: No changes'); + } } catch (e) { console.warn('[DBSyncManager] Background refresh failed:', e); + } finally { + this.isRefreshingList = false; } } From 412336722fb9d32988f54bd3bc39133c77e89853 Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Mon, 15 Dec 2025 13:52:27 +0100 Subject: [PATCH 5/5] docs: add escalation reports and update storage documentation Add comprehensive escalation handoff reports documenting two critical issues: - Sidebar conversation loading from database (resolved) - Timestamp updates when viewing conversations (unresolved) Update StorageAndPersistence.md with current system status, working features table, and known issues. Disable Google Drive sync UI in ConversationSidebar (backend remains available). Implement message count tracking in useBranchingChat to prevent timestamp updates on view-only operations. Improve db-sync-manager to detect and fix userId mismatches during conversation sync. --- docs/escalation_report.md | 415 +++++++++++++++++ docs/escalation_report_timestamp_issue.md | 427 ++++++++++++++++++ .../john-gpt/StorageAndPersistence.md | 41 +- .../components/ConversationSidebar.tsx | 6 +- .../john-gpt/hooks/useBranchingChat.ts | 23 +- src/lib/storage/db-sync-manager.ts | 13 +- 6 files changed, 909 insertions(+), 16 deletions(-) create mode 100644 docs/escalation_report.md create mode 100644 docs/escalation_report_timestamp_issue.md diff --git a/docs/escalation_report.md b/docs/escalation_report.md new file mode 100644 index 0000000..77235d6 --- /dev/null +++ b/docs/escalation_report.md @@ -0,0 +1,415 @@ +# Escalation Handoff Report + +**Generated:** 2025-12-15T10:41:00+01:00 +**Original Issue:** JohnGPT Conversation Sidebar Not Loading Conversations from Database + +--- + +## PART 1: THE DAMAGE REPORT + +### 1.1 Original Goal +Fix the JohnGPT conversation sidebar to display all conversations from the database. The API correctly returns 8 conversations via `GET /api/conversations`, but the sidebar only shows 1 conversation (the currently active one). + +### 1.2 Observed Failure / Error +``` +Network Tab shows: +GET /api/conversations 200 in 2277ms +- Returns 8 conversations correctly + +But sidebar only displays 1 conversation under "TODAY" + +Console logs show: +[DBSyncManager] refreshConversationList: Fetching from API... +[DBSyncManager] refreshConversationList: API returned 8 conversations +[DBSyncManager] refreshConversationList: No changes +``` + +The "No changes" message indicates conversations are NOT being cached because the code thinks they already exist. But when `indexedDBClient.listConversations(userId)` is called, it returns an empty array (or only 1 item). + +### 1.3 Failed Approach +1. **Fixed infinite refresh loop** - Added `isRefreshingList` flag to prevent recursive API calls ✅ +2. **Fixed 400 Bad Request on sync** - Changed `personaId: null` to `undefined` (Zod accepts undefined, not null) ✅ +3. **Attempted to fix userId mapping** - Modified `mapApiToCache()` to include `userId` from the manager's state since API response doesn't include it + +The userId fix was supposed to work because: +- API response doesn't include `userId` field in conversations +- `mapApiToCache()` was setting `userId: apiConv.userId` which is `undefined` +- IndexedDB's `listConversations(userId)` filters by the `userId` index +- With `userId: undefined`, conversations weren't being found + +**But it still doesn't work after the fix.** + +### 1.4 Key Files Involved +- `src/lib/storage/db-sync-manager.ts` - Main sync orchestrator +- `src/lib/storage/indexeddb-client.ts` - IndexedDB client with conversation storage +- `src/features/john-gpt/components/ConversationSidebar.tsx` - UI component that displays conversations +- `src/app/api/conversations/route.ts` - API endpoint that lists conversations + +### 1.5 Best-Guess Diagnosis +The remaining issue is likely one of: + +1. **Race condition**: `listConversations` is called BEFORE `refreshConversationList` completes caching +2. **IndexedDB not flushing**: Transactions may not be committing before the next read +3. **userId mismatch**: The `userId` being passed to `listConversations` might differ from what's stored +4. **Cache comparison bug**: In `refreshConversationList`, the `!cached` check on line 537 might be returning truthy when it should be falsy + +**Specific debug points to check:** +- What does `indexedDBClient.getConversation(serverConv.id)` return in `refreshConversationList`? +- After `updateCache(mapped, false)` is called, does `indexedDBClient.listConversations(userId)` immediately return the new data? +- Is `this.userId` in `mapApiToCache` correctly set (non-null)? + +--- + +## PART 2: FULL FILE CONTENTS (Self-Contained) + +### File: `src/lib/storage/db-sync-manager.ts` +```typescript +/** + * DB Sync Manager for JohnGPT Conversations + * + * Orchestrates synchronization between: + * - IndexedDB (local cache, instant access) + * - Neon Database (via API, source of truth) + * + * Features: + * - Debounced saves (5-second delay to reduce API calls) + * - Offline queue (syncs when network reconnects) + * - Background sync (Web Background Sync API logic) + * - Event emitters for UI feedback + * - Guest support: Only uses IndexedDB, no API calls + */ + +import { indexedDBClient, type CachedConversation } from './indexeddb-client'; +import { googleDriveClient, type ConversationFile } from './google-drive-client'; +import type { UIMessage } from '@ai-sdk/react'; + +// ============================================================================ +// Types +// ============================================================================ + +export type SyncStatus = 'idle' | 'syncing' | 'synced' | 'error' | 'offline'; + +export type SyncEvent = { + conversationId: string; + status: SyncStatus; + error?: string; + timestamp: number; +}; + +type SyncListener = (event: SyncEvent) => void; + +type DebouncedSave = { + conversationId: string; + timeoutId: NodeJS.Timeout; + scheduledAt: number; + isWidget?: boolean; +}; + +// ============================================================================ +// Constants +// ============================================================================ + +const DEBOUNCE_DELAY_MS = 5000; // 5 seconds +const NETWORK_CHECK_INTERVAL = 10000; // 10 seconds + +// ============================================================================ +// DB Sync Manager Class +// ============================================================================ + +export class DBSyncManager { + private listeners: SyncListener[] = []; + private listListeners: (() => void)[] = []; + private debouncedSaves: Map = new Map(); + private syncStatus: Map = new Map(); + private isOnline: boolean = typeof navigator !== 'undefined' ? navigator.onLine : true; + private networkCheckInterval: NodeJS.Timeout | null = null; + private isAuthenticated: boolean = false; + private userId: string | null = null; + private isDriveConnected: boolean = false; + private isRefreshingList: boolean = false; + + constructor() { + if (typeof window !== 'undefined') { + // Listen for online/offline events + window.addEventListener('online', this.handleOnline.bind(this)); + window.addEventListener('offline', this.handleOffline.bind(this)); + + // Periodic network check (some browsers don't fire events reliably) + this.startNetworkCheck(); + } + } + + // ========================================================================== + // Public API + // ========================================================================== + + /** + * Initialize with user state + */ + initialize(userId: string | null) { + this.userId = userId; + // Authenticated if userId exists AND is not anonymous + this.isAuthenticated = !!userId && !userId.startsWith('anonymous-'); + + console.log(`[DBSyncManager] Initialized. User: ${userId || 'Guest'} (Auth: ${this.isAuthenticated}), Online: ${this.isOnline}`); + + if (this.isAuthenticated && this.isOnline) { + this.processOfflineQueue(); + } + } + + /** + * Initialize Google Drive connection + * Fetches token from API and sets it on client + */ + async initializeGoogleDrive(userId: string): Promise { + if (!userId || userId.startsWith('anonymous-')) return false; + + try { + const res = await fetch(`/api/user/drive-config?userId=${userId}`); + if (res.status === 404) { + this.isDriveConnected = false; + return false; + } + if (!res.ok) throw new Error('Failed to fetch Drive config'); + + const { accessToken } = await res.json(); + googleDriveClient.setAccessToken(accessToken); + this.isDriveConnected = true; + console.log('[DBSyncManager] Google Drive connected'); + return true; + } catch (error) { + console.warn('[DBSyncManager] Failed to initialize Google Drive:', error); + this.isDriveConnected = false; + return false; + } + } + + /** + * List all conversations (from cache + API) + * If cache is empty and authenticated, fetches from API first (fresh window scenario) + */ + async listConversations(userId: string): Promise { + // 1. Get from cache (instant) + const cached = await indexedDBClient.listConversations(userId); + + // 2. If cache is empty and we're online+authenticated, await API fetch first + // This handles the "new browser window" scenario where IndexedDB is empty + if (cached.length === 0 && this.isOnline && this.isAuthenticated) { + console.log('[DBSyncManager] Cache empty, fetching from API...'); + await this.refreshConversationList(userId); + // Return the now-populated cache + return await indexedDBClient.listConversations(userId); + } + + // 3. Background refresh for non-empty cache (stale-while-revalidate) + // Only if not already refreshing (prevents infinite loop) + if (this.isOnline && this.isAuthenticated && !this.isRefreshingList) { + this.refreshConversationList(userId).catch((error) => { + console.warn('[DBSyncManager] Background refresh failed:', error); + }); + } + + return cached; + } + + // ========================================================================== + // Private Methods - Helpers + // ========================================================================== + + private mapApiToCache(apiConv: any, overrideUserId?: string): CachedConversation { + return { + conversationId: apiConv.id, + // API list endpoint doesn't return userId, so use override or fallback to manager's userId + userId: overrideUserId || apiConv.userId || this.userId, + title: apiConv.title, + createdAt: apiConv.createdAt, + updatedAt: apiConv.updatedAt, + messages: apiConv.messages as any[] || [], + lastSyncedAt: Date.now(), // fresh from API + isDirty: 0, + localVersion: apiConv.localVersion, + personaId: apiConv.personaId, + selectedModelId: apiConv.selectedModelId, + // Legacy/Drive fields + driveFileId: apiConv.driveFileId, + } as any; + } + + private async updateCache(conversation: CachedConversation, isDirty: boolean): Promise { + const cached: CachedConversation = { + ...conversation, + lastSyncedAt: Date.now(), + isDirty: isDirty ? 1 : 0, + }; + await indexedDBClient.saveConversation(cached); + } + + private async refreshConversationList(userId: string): Promise { + // Prevent concurrent/recursive refreshes + if (this.isRefreshingList) { + console.log('[DBSyncManager] refreshConversationList: Already refreshing, skipping'); + return; + } + + this.isRefreshingList = true; + + try { + console.log('[DBSyncManager] refreshConversationList: Fetching from API...'); + const res = await fetch('/api/conversations'); + if (!res.ok) { + console.warn('[DBSyncManager] refreshConversationList: API returned', res.status); + return; + } + + const serverConversations = await res.json(); + console.log(`[DBSyncManager] refreshConversationList: API returned ${Array.isArray(serverConversations) ? serverConversations.length : 0} conversations`); + + let hasChanges = false; + + if (Array.isArray(serverConversations)) { + for (const serverConv of serverConversations) { + const cached = await indexedDBClient.getConversation(serverConv.id); + const serverTime = new Date(serverConv.updatedAt).getTime(); + + // If not in cache, or server is newer - save to cache + if (!cached || serverTime > new Date(cached.updatedAt).getTime()) { + const mapped = this.mapApiToCache(serverConv, userId); + await this.updateCache(mapped, false); + hasChanges = true; + console.log(`[DBSyncManager] Cached conversation: ${serverConv.id} (${serverConv.title})`); + } + } + } + + // Only notify if there were actual changes (prevents infinite loop) + if (hasChanges) { + console.log('[DBSyncManager] refreshConversationList: Changes found, notifying listeners'); + this.notifyListListeners(); + } else { + console.log('[DBSyncManager] refreshConversationList: No changes'); + } + } catch (e) { + console.warn('[DBSyncManager] Background refresh failed:', e); + } finally { + this.isRefreshingList = false; + } + } + + // ... (remaining methods omitted for brevity - see full file) +} + +export const dbSyncManager = new DBSyncManager(); +``` + +### File: `src/lib/storage/indexeddb-client.ts` (Key Parts) +```typescript +/** + * List all cached conversations for a user + * Returns sorted by most recently updated + */ +async listConversations(userId: string): Promise { + await this.init(); + if (!this.db) throw new Error('IndexedDB not initialized'); + + return new Promise((resolve, reject) => { + const transaction = this.db!.transaction([STORE_CONVERSATIONS], 'readonly'); + const store = transaction.objectStore(STORE_CONVERSATIONS); + const index = store.index('userId'); + + const request = index.getAll(userId); // <-- FILTERS BY userId INDEX + + request.onsuccess = () => { + const conversations = request.result || []; + // Sort by updatedAt descending (most recent first) + conversations.sort((a, b) => { + const dateA = new Date(a.updatedAt).getTime(); + const dateB = new Date(b.updatedAt).getTime(); + return dateB - dateA; + }); + resolve(conversations); + }; + + request.onerror = () => reject(request.error); + }); +} +``` + +### File: `src/features/john-gpt/components/ConversationSidebar.tsx` (Key Parts) +```typescript +// Initialize DB & Fetch conversations +useEffect(() => { + dbSyncManager.initialize(user.id); + + const fetchConversations = async () => { + try { + // Use DB Sync Manager (Fast + Offline) + const cachedDocs = await dbSyncManager.listConversations(user.id); + + // Transform response + const transformed = cachedDocs.map((conv) => ({ + id: conv.conversationId, + title: conv.title || 'New Chat', + date: conv.updatedAt, + preview: conv.title || 'No preview', + })); + + // Sort by date desc (if not already) + transformed.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); + + setConversations(transformed); + } catch (error) { + console.error('[ConversationSidebar] Failed to load conversations:', error); + } finally { + setIsLoading(false); + } + }; + + // Initial fetch + fetchConversations(); + + // Subscribe to ANY list changes (Created, Saved, Deleted) + const unsubscribe = dbSyncManager.onListChange(() => { + console.log('[ConversationSidebar] List changed, refreshing...'); + fetchConversations(); + }); + + return () => { + unsubscribe(); + }; +}, [user.id]); +``` + +--- + +## PART 3: DIRECTIVE FOR ORCHESTRATOR + +**Attention: Senior AI Orchestrator** + +You have received this Escalation Handoff Report. A local agent has failed to solve this problem. + +**Your Directive:** + +1. **Analyze the Failure:** Based on Part 1 (the report) and Part 2 (the code), diagnose the TRUE root cause. The key question is: **Why does `indexedDBClient.listConversations(userId)` return empty/incomplete data after `refreshConversationList` has called `updateCache` for 8 conversations?** + +2. **Debug Points to Investigate:** + - Add logging inside `indexedDBClient.saveConversation()` to confirm it's being called with correct `userId` + - Add logging inside `indexedDBClient.listConversations()` to see what `userId` is being queried + - Check if `this.userId` in `dbSyncManager` is correctly set before `refreshConversationList` runs + - Verify the IndexedDB transactions are committing (check if `transaction.oncomplete` is firing) + +3. **Suspected Issues:** + - The `userId` passed to `listConversations` might be `user_01KB92E8H0QA9RPPSBP4JPS3EA` but stored as something else + - The IndexedDB index query might not be working as expected + - There might be an async timing issue between save and read + +4. **Recommended Fix Strategy:** + - Add extensive debug logging to both `indexedDBClient.saveConversation` and `indexedDBClient.listConversations` + - Log the exact `userId` being stored and queried + - Use browser DevTools → Application → IndexedDB to manually inspect the stored data + - Check if conversations are being stored at all, and what their `userId` field contains + +5. **Nuclear Option:** + - If the IndexedDB filtering is too complex, consider NOT filtering by userId in the client (since the API already filters by authenticated user), and just return all cached conversations + +**Begin your analysis now.** diff --git a/docs/escalation_report_timestamp_issue.md b/docs/escalation_report_timestamp_issue.md new file mode 100644 index 0000000..88386cf --- /dev/null +++ b/docs/escalation_report_timestamp_issue.md @@ -0,0 +1,427 @@ +# Escalation Handoff Report + +**Generated:** 2025-12-15T13:17:00+01:00 +**Original Issue:** JohnGPT Conversation Timestamp Updates When Simply Viewing (Not Modifying) + +--- + +## PART 1: THE DAMAGE REPORT + +### 1.1 Original Goal +Prevent the conversation `updatedAt` timestamp from being updated when a user simply clicks on and views an old conversation. The timestamp should only update when the user actually sends a new message or modifies the conversation. + +### 1.2 Observed Failure / Error +When clicking on an old conversation (e.g., from "Yesterday" or "Previous 7 Days" group), it immediately jumps to the "Today" group in the sidebar. The terminal shows: + +``` +PATCH /api/conversations/bc455ffe-42f4-4326-af5d-63b9e220e63f 200 in 498ms +``` + +This PATCH request is being sent even though the user did NOT modify the conversation - they only viewed it. + +### 1.3 Failed Approach + +**Approach 1: Boolean Flag (Failed)** +- Added `messagesJustLoadedRef` boolean flag +- Set to `true` when `setMessages` is called externally (loading conversation) +- In save effect, skip save if flag is `true`, then reset to `false` +- **WHY IT FAILED:** The save effect runs MULTIPLE times due to different dependency changes (`messages.length`, `tree`, `chatHelpers.status`). The first run resets the flag to `false`, but subsequent runs see it as `false` and proceed to save. + +**Approach 2: Message Count Comparison (Failed)** +- Added `loadedMessageCountRef` to store the count of loaded messages +- In save effect, skip if `messages.length <= loadedMessageCountRef.current` +- **WHY IT FAILED:** Still triggers PATCH. Possible reasons: + 1. The count comparison might not account for all code paths + 2. There may be OTHER places calling `dbSyncManager.saveConversation` directly + 3. The `revalidateFromApi` in `loadConversation` might be triggering updates + +### 1.4 Key Files Involved +- `src/features/john-gpt/hooks/useBranchingChat.ts` - Main chat hook with save logic +- `src/features/john-gpt/components/ChatView.tsx` - Component that loads conversations +- `src/lib/storage/db-sync-manager.ts` - Sync manager that saves to API +- `src/features/john-gpt/hooks/useConversationPersistence.ts` - Persistence hook + +### 1.5 Best-Guess Diagnosis + +The root cause is likely: + +1. **Multiple Save Triggers:** The `useBranchingChat` save effect has dependencies `[messages.length, chatHelpers.status, tree, conversationId, userId, options.isWidget, options.modelId]`. When loading a conversation, `tree` state changes multiple times as messages are processed, triggering the save effect even after the count check passes. + +2. **Tree State Sync:** Lines 175-248 sync messages to tree state. This happens AFTER the messages are set, and causes additional effect reruns that bypass the load check. + +3. **Possible Race Condition:** The wrapped `setMessages` sets `loadedMessageCountRef`, but by the time the save effect runs, `chatHelpers.status` or `tree` may have changed, causing the effect to fire again when the ref has already been "used up." + +**Debug Points to Investigate:** +- Add logging to see EXACTLY how many times the save effect runs after loading a conversation +- Check if `tree` changes trigger the save after the count check has passed +- Check if there are direct calls to `dbSyncManager.saveConversation` outside of `useBranchingChat` + +--- + +## PART 2: FULL FILE CONTENTS (Self-Contained) + +### File: `src/features/john-gpt/hooks/useBranchingChat.ts` +```typescript +import { useState, useCallback, useEffect, useMemo, useRef } from 'react'; +import { useChat, UIMessage } from '@ai-sdk/react'; +import { useRouter, usePathname } from 'next/navigation'; +import { dbSyncManager } from '@/lib/storage/db-sync-manager'; + +export type BranchingMessage = UIMessage & { + parentId?: string | null; + childrenIds?: string[]; + branchIndex?: number; + branchCount?: number; +}; + +type TreeNode = { + id: string; + message: UIMessage; + parentId: string | null; + childrenIds: string[]; + createdAt: number; +}; + +type MessageTree = Record; + +export type UseBranchingChatOptions = { + conversationId?: string; + userId?: string; + api?: string; + body?: any; + onFinish?: any; + isWidget?: boolean; + scrollToSection?: (sectionId: string) => void; + modelId?: string | null; +}; + +export function useBranchingChat(options: UseBranchingChatOptions = {}) { + const { conversationId, userId } = options; + const router = useRouter(); + const pathname = usePathname(); + + // Initialize SyncManager + useEffect(() => { + dbSyncManager.initialize(userId || null); + }, [userId]); + + // 1. Internal Tree State + const [tree, setTree] = useState({}); + const [headId, setHeadId] = useState(null); + + // Track if messages were just loaded externally (not user-modified) + // This prevents saving (and updating timestamp) when simply viewing a conversation + // We store the COUNT of loaded messages - only save when new messages are ADDED + const loadedMessageCountRef = useRef(null); + + // Track the "active" child for each node to restore history correctly when navigating back + const [activePathMap, setActivePathMap] = useState>({}); + + const [currentMode, setCurrentMode] = useState(null); + + // 2. Initialize useChat with currentPath for navigation context + console.log('[useBranchingChat] Options received:', { body: options.body, api: options.api, currentPath: pathname }); + + const chatHelpers = useChat({ + ...options, + // @ts-expect-error - body is supported but types are strict + body: { ...options.body, currentPath: pathname }, + onFinish: (response: any) => { + const msg = response.message || response; + + if (msg?.parts) { + for (const part of msg.parts) { + if (part.type === 'tool-goTo' && part.state === 'output-available') { + const result = part.output; + const scrollFn = options.scrollToSection; + + switch (result?.action) { + case 'navigate': + setTimeout(() => { + console.log('[goTo] Navigating to:', result.url); + const separator = result.url.includes('?') ? '&' : '?'; + router.push(`${result.url}${separator}spotlight=page`); + }, 1500); + break; + + case 'scrollToSection': + if (result.sectionId && scrollFn) { + setTimeout(() => { + console.log('[goTo] Scrolling to:', result.sectionId); + scrollFn(result.sectionId); + }, 500); + } + break; + + case 'navigateAndScroll': + setTimeout(() => { + console.log('[goTo] Navigate + Scroll:', result.url, result.sectionId); + const sep = result.url.includes('?') ? '&' : '?'; + router.push(`${result.url}${sep}spotlight=${result.sectionId}`); + }, 1500); + break; + } + break; + } + } + } + + if (options.onFinish) { + options.onFinish(response); + } + }, + }) as any; + + const { messages, setMessages: originalSetMessages, sendMessage } = chatHelpers; + + // Wrap setMessages to track when messages are loaded externally vs user-modified + // This prevents saving (and updating timestamp) when simply viewing a conversation + const setMessages = useCallback((msgs: any) => { + // Store the count of externally loaded messages + loadedMessageCountRef.current = Array.isArray(msgs) ? msgs.length : 0; + originalSetMessages(msgs); + }, [originalSetMessages]); + + // 🚀 Dynamic model selection wrapper + const sendMessageWithModel = useCallback( + async (message: any, modelId?: string | null) => { + return sendMessage(message, { + body: { modelId }, + }); + }, + [sendMessage] + ); + + // Listen for message updates to set the mode from metadata + useEffect(() => { + if (!messages || messages.length === 0) return; + if (chatHelpers.status === 'streaming') return; + + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role === 'assistant' && (lastMessage as any).metadata) { + const mode = (lastMessage as any).metadata.mode; + if (mode) { + console.log('[useBranchingChat] Found mode in metadata:', mode); + setCurrentMode(mode); + } + } + }, [messages, chatHelpers.status]); + + // Helper to reconstruct path from a given leaf/head ID + const getPathToNode = useCallback((leafId: string, currentTree: MessageTree): UIMessage[] => { + const path: UIMessage[] = []; + let currentId: string | null = leafId; + + while (currentId && currentTree[currentId]) { + path.unshift(currentTree[currentId].message); + currentId = currentTree[currentId].parentId; + } + return path; + }, []); + + // 3. Sync `messages` from useChat to `tree` (for streaming updates) + useEffect(() => { + if (messages.length === 0) return; + + const status = chatHelpers.status; + if (status === 'streaming') { + return; + } + + const lastMsg = messages[messages.length - 1] as any; + + setTree(prevTree => { + const existingNode = prevTree[lastMsg.id]; + + if (existingNode) { + const existingContent = (existingNode.message as any).content; + const newContent = lastMsg.content; + + if (existingContent === newContent && + (existingNode.message as any).toolInvocations === lastMsg.toolInvocations) { + return prevTree; + } + return { + ...prevTree, + [lastMsg.id]: { ...existingNode, message: lastMsg } + }; + } + + let newParentId = headId; + + if (messages.length > 1) { + const parentMsg = messages[messages.length - 2]; + if (prevTree[parentMsg.id]) { + newParentId = parentMsg.id; + } + } else { + newParentId = null; + } + + const newNode: TreeNode = { + id: lastMsg.id, + message: lastMsg, + parentId: newParentId || null, + childrenIds: [], + createdAt: Date.now(), + }; + + const nextTree: MessageTree = { ...prevTree, [lastMsg.id]: newNode }; + + if (newParentId && prevTree[newParentId] && !prevTree[newParentId].childrenIds.includes(lastMsg.id)) { + nextTree[newParentId] = { + ...prevTree[newParentId], + childrenIds: [...prevTree[newParentId].childrenIds, lastMsg.id] + }; + } + + return nextTree; + }); + + if (lastMsg.id !== headId) { + setHeadId(lastMsg.id); + + if (messages.length > 1) { + const parentId = messages[messages.length - 2].id; + setActivePathMap(prev => ({ ...prev, [parentId]: lastMsg.id })); + } + } + + }, [messages, headId, chatHelpers.status]); + + // Initialize SyncManager + useEffect(() => { + dbSyncManager.initialize(userId || null); + }, [userId]); + + // 3.5. Persistence: Save to IndexedDB (and queue DB sync) + // PERFORMANCE: Skip saves during active streaming to prevent UI freezing + useEffect(() => { + // Only save if we have a conversation ID and user ID and messages + if (!conversationId || !userId || messages.length === 0) return; + + // Skip save if messages count matches what was loaded (no new messages added) + // This prevents updating timestamp when simply viewing a conversation + if (loadedMessageCountRef.current !== null && messages.length <= loadedMessageCountRef.current) { + return; + } + + // Skip save during active streaming - only save when streaming completes + const currentStatus = chatHelpers.status; + if (currentStatus === 'streaming' || currentStatus === 'submitted') { + return; + } + + const saveConversation = async () => { + try { + const messagesToSave = messages.map((msg: any) => { + const node = tree[msg.id]; + return { + ...msg, + parentId: node?.parentId || null, + childrenIds: node?.childrenIds || [], + createdAt: new Date(node?.createdAt || Date.now()).toISOString(), + }; + }); + + let title = 'New Chat'; + if (messages.length >= 2) { + const firstUserMsg = messages.find((m: any) => m.role === 'user'); + if (firstUserMsg?.parts) { + const textPart = firstUserMsg.parts.find((p: any) => p.type === 'text'); + if (textPart && 'text' in textPart) { + title = textPart.text.slice(0, 50); + if (textPart.text.length > 50) title += '...'; + } + } + } + + await dbSyncManager.saveConversation( + conversationId, + userId, + title, + messagesToSave, + { + isWidget: options.isWidget, + selectedModelId: options.modelId || undefined + } + ); + + console.log('[useBranchingChat] Conversation saved to IndexedDB:', conversationId); + } catch (error) { + console.error('[useBranchingChat] Save failed:', error); + } + }; + + const debounceTimer = setTimeout(saveConversation, 500); + return () => clearTimeout(debounceTimer); + }, [messages.length, chatHelpers.status, tree, conversationId, userId, options.isWidget, options.modelId]); + + // ... remaining code for editMessage, navigateBranch, etc. + + return { + ...chatHelpers, + messages: messagesWithBranches, + setMessages, // Export wrapped version + editMessage, + navigateBranch, + currentMode, + sendMessageWithModel, + }; +} +``` + +### File: `src/features/john-gpt/components/ChatView.tsx` (Relevant Section) +```typescript +// Load conversation on mount if conversationId exists +useEffect(() => { + if (!internalConversationId || messages.length > 0 || importSessionId) return; + + const loadExistingConversation = async () => { + try { + const conversation = await loadConversation(internalConversationId); + + if (conversation && conversation.messages.length > 0) { + // Hydrate messages into chat + setMessages(conversation.messages as any); + console.log('[ChatView] Loaded conversation:', internalConversationId, conversation.messages.length, 'messages'); + } + } catch (error) { + console.error('[ChatView] Failed to load conversation:', error); + } + }; + + loadExistingConversation(); +}, [internalConversationId, loadConversation, setMessages, messages.length, importSessionId]); +``` + +--- + +## PART 3: DIRECTIVE FOR ORCHESTRATOR + +**Attention: Senior AI Orchestrator** + +You have received this Escalation Handoff Report. A local agent has failed to solve this problem. + +**Your Directive:** + +1. **Analyze the Failure:** The core issue is that the save effect in `useBranchingChat` is being triggered when viewing (not modifying) a conversation. The attempted fixes (boolean flag, count comparison) both failed because of React's effect re-run behavior. + +2. **Key Investigation Points:** + - WHY does the save effect run after the count comparison passes? Is it the `tree` dependency? + - Add console.log INSIDE the save effect to trace: when it runs, what the counts are, and what triggers it + - Check if the `tree` state update from lines 175-248 is causing an additional effect run AFTER the count check + - Consider if the solution should be at a different level (e.g., in `dbSyncManager.saveConversation` itself) + +3. **Alternative Solution Approaches:** + - **Option A:** Remove `tree` from the save effect dependencies - only save on actual `messages.length` changes + - **Option B:** Track MESSAGE IDs instead of counts - only save if there are NEW message IDs not in the loaded set + - **Option C:** Add a debounce/stable ref that tracks "has user interacted" vs "just loaded" + - **Option D:** Move the "should save" logic to `dbSyncManager.saveConversation` itself, comparing against the last saved state + +4. **Execute or Hand Off:** Implement the correct fix and verify with test scenario: + - Load an old conversation (from "Yesterday" or "Older" group) + - Verify NO PATCH request is sent + - Verify the conversation stays in its original date group + +**Begin your analysis now.** diff --git a/docs/features/john-gpt/StorageAndPersistence.md b/docs/features/john-gpt/StorageAndPersistence.md index d3f0654..d855247 100644 --- a/docs/features/john-gpt/StorageAndPersistence.md +++ b/docs/features/john-gpt/StorageAndPersistence.md @@ -13,16 +13,40 @@ The system uses a three-tier storage strategy: * **Benefit:** Instant load times, offline support, zero latency. * **Library:** `idb` (via `IndexedDBClient`). -2. **Google Drive (Cloud Backup):** +2. **Neon PostgreSQL (Server DB):** + * **Role:** Primary cloud storage for conversations with API sync. + * **Benefit:** Cross-browser sync, persistent storage, real-time availability. + * **Manager:** `DBSyncManager` (`src/lib/storage/db-sync-manager.ts`) + * **Data:** Full conversation metadata and messages stored via Prisma. + +3. **Google Drive (Optional Cloud Backup):** + * **Status:** Backend available but **UI disabled** (2025-12-15). * **Role:** Long-term storage and cross-device sync. * **Benefit:** User owns their data, accessible outside the app. * **Format:** JSON files named `[AI Title] - [8-char ID].json`. * **Library:** Custom `GoogleDriveClient` using Google Drive API v3. -3. **Neon DB (Metadata):** - * **Role:** Lightweight index for the conversation sidebar. - * **Benefit:** Fast listing of conversations without scanning Drive files. - * **Data:** `id`, `title`, `createdAt`, `updatedAt`, `driveFileId`. +## 2.1 Current Status (2025-12-15) + +### ✅ Working Features + +| Feature | Status | Notes | +|---------|--------|-------| +| **Sidebar loads all conversations** | ✅ Working | Lists all user conversations from DB | +| **Messages load correctly** | ✅ Working | Full message history displays when opening a conversation | +| **New conversations save to DB** | ✅ Working | Auto-syncs with 5s debounce | +| **IndexedDB caching** | ✅ Working | Instant loads, offline support | +| **Cross-browser sync** | ✅ Working | Conversations sync via Neon PostgreSQL | +| **AI title generation** | ✅ Working | Triggers after 6 messages | +| **Conversation deletion** | ✅ Working | Removes from DB and cache | +| **Offline queue** | ✅ Working | Pending syncs retry when online | + +### ⚠️ Known Issues + +| Issue | Severity | Ticket | +|-------|----------|--------| +| Viewing old conversations updates `updatedAt` timestamp | Low | See `docs/escalation_report_timestamp_issue.md` | +| Google Drive sync UI disabled | Info | Backend works, UI hidden from sidebar | ## 3. Key Components @@ -137,5 +161,12 @@ The `MessagePartSchema` (`src/features/john-gpt/schema.ts`) validates AI SDK mes | Date | Change | |------|--------| +| 2025-12-15 | **FIX:** Sidebar now correctly displays all conversations by adding `userId` mismatch check in `refreshConversationList` | +| 2025-12-15 | **FIX:** Sync 400 Bad Request error resolved - Zod `.optional()` accepts `undefined` but not `null` | +| 2025-12-15 | **FIX:** Infinite refresh loop prevented with `isRefreshingList` flag | +| 2025-12-15 | **FIX:** Individual conversation loading now fetches from API if cache has 0 messages (metadata-only) | +| 2025-12-15 | Disabled Google Drive sync UI from sidebar (backend still available, UI hidden) | +| 2025-12-15 | **KNOWN ISSUE:** Viewing old conversations updates their `updatedAt` timestamp - see `docs/escalation_report_timestamp_issue.md` | | 2024-12-14 | Hardened `MessagePartSchema` with explicit discriminated union (was `.passthrough()`) | | 2024-12-14 | Added ownership checks (403 responses) to `/api/conversations/[id]` | + diff --git a/src/features/john-gpt/components/ConversationSidebar.tsx b/src/features/john-gpt/components/ConversationSidebar.tsx index 2b9d9c3..13d3375 100644 --- a/src/features/john-gpt/components/ConversationSidebar.tsx +++ b/src/features/john-gpt/components/ConversationSidebar.tsx @@ -247,8 +247,8 @@ export function ConversationSidebar({ user, isDriveConnected, className, activeC - {/* Drive Connection Status */} - {!isDriveConnected ? ( + {/* Google Drive sync disabled - not currently used */} + {/* {!isDriveConnected ? (

@@ -285,7 +285,7 @@ export function ConversationSidebar({ user, isDriveConnected, className, activeC

- )} + )} */} {/* Search */}
diff --git a/src/features/john-gpt/hooks/useBranchingChat.ts b/src/features/john-gpt/hooks/useBranchingChat.ts index 38f497b..798b050 100644 --- a/src/features/john-gpt/hooks/useBranchingChat.ts +++ b/src/features/john-gpt/hooks/useBranchingChat.ts @@ -1,4 +1,4 @@ -import { useState, useCallback, useEffect, useMemo } from 'react'; +import { useState, useCallback, useEffect, useMemo, useRef } from 'react'; import { useChat, UIMessage } from '@ai-sdk/react'; import { useRouter, usePathname } from 'next/navigation'; import { dbSyncManager } from '@/lib/storage/db-sync-manager'; @@ -45,6 +45,11 @@ export function useBranchingChat(options: UseBranchingChatOptions = {}) { const [tree, setTree] = useState({}); const [headId, setHeadId] = useState(null); + // Track if messages were just loaded externally (not user-modified) + // This prevents saving (and updating timestamp) when simply viewing a conversation + // We store the COUNT of loaded messages - only save when new messages are ADDED + const loadedMessageCountRef = useRef(null); + // Track the "active" child for each node to restore history correctly when navigating back const [activePathMap, setActivePathMap] = useState>({}); @@ -115,7 +120,15 @@ export function useBranchingChat(options: UseBranchingChatOptions = {}) { }, }) as any; - const { messages, setMessages, sendMessage } = chatHelpers; + const { messages, setMessages: originalSetMessages, sendMessage } = chatHelpers; + + // Wrap setMessages to track when messages are loaded externally vs user-modified + // This prevents saving (and updating timestamp) when simply viewing a conversation + const setMessages = useCallback((msgs: any) => { + // Store the count of externally loaded messages + loadedMessageCountRef.current = Array.isArray(msgs) ? msgs.length : 0; + originalSetMessages(msgs); + }, [originalSetMessages]); // 🚀 Dynamic model selection wrapper // useChat memoizes `body` at init, so we pass modelId per-request via sendMessage options @@ -245,6 +258,12 @@ export function useBranchingChat(options: UseBranchingChatOptions = {}) { // Only save if we have a conversation ID and user ID and messages if (!conversationId || !userId || messages.length === 0) return; + // Skip save if messages count matches what was loaded (no new messages added) + // This prevents updating timestamp when simply viewing a conversation + if (loadedMessageCountRef.current !== null && messages.length <= loadedMessageCountRef.current) { + return; + } + // Skip save during active streaming - only save when streaming completes // This prevents constant saves during token-by-token updates const currentStatus = chatHelpers.status; diff --git a/src/lib/storage/db-sync-manager.ts b/src/lib/storage/db-sync-manager.ts index 8b60b64..1e06544 100644 --- a/src/lib/storage/db-sync-manager.ts +++ b/src/lib/storage/db-sync-manager.ts @@ -533,22 +533,23 @@ export class DBSyncManager { const cached = await indexedDBClient.getConversation(serverConv.id); const serverTime = new Date(serverConv.updatedAt).getTime(); - // If not in cache, or server is newer - save to cache - if (!cached || serverTime > new Date(cached.updatedAt).getTime()) { + // If not in cache, or server is newer, or userId is wrong/missing - save to cache + const needsUpdate = !cached || + serverTime > new Date(cached.updatedAt).getTime() || + cached.userId !== userId; + + if (needsUpdate) { const mapped = this.mapApiToCache(serverConv, userId); await this.updateCache(mapped, false); hasChanges = true; - console.log(`[DBSyncManager] Cached conversation: ${serverConv.id} (${serverConv.title})`); } } } // Only notify if there were actual changes (prevents infinite loop) if (hasChanges) { - console.log('[DBSyncManager] refreshConversationList: Changes found, notifying listeners'); + console.log(`[DBSyncManager] Synced ${serverConversations.length} conversations from API`); this.notifyListListeners(); - } else { - console.log('[DBSyncManager] refreshConversationList: No changes'); } } catch (e) { console.warn('[DBSyncManager] Background refresh failed:', e);