feat: add phase 2 notebook search/chunk routes and oss roadmap docs

Author: claude

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install
+        run: npm ci
+
+      - name: Typecheck
+        run: npm run type-check
+
+      - name: Test
+        run: npm test
+
+  smoke-v2-live:
+    runs-on: ubuntu-latest
+    needs: validate
+    if: ${{ secrets.MINDSPRING_BASE_URL != '' && secrets.MINDSPRING_API_KEY != '' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install
+        run: npm ci
+
+      - name: Run v2 smoke against deployed worker
+        env:
+          MINDSPRING_BASE_URL: ${{ secrets.MINDSPRING_BASE_URL }}
+          MINDSPRING_API_KEY: ${{ secrets.MINDSPRING_API_KEY }}
+        run: npm run smoke:v2

ARCHITECTURE.md

@@ -0,0 +1,52 @@
+# MindSpring Architecture
+
+## Runtime Model
+
+- Cloudflare Worker (Hono) serves API + static SPA
+- Queue consumer handles asynchronous ingestion
+- Data planes:
+  - Vectorize: embeddings + lightweight metadata
+  - KV: fast hydration + request telemetry + v1 progress states
+  - D1: v2 relational notebook/source/chunk/job state
+  - R2: raw uploaded source files
+
+## v1 vs v2 Boundaries
+
+### v1 (`/api/*`)
+
+- Primary primitive: conversation archive
+- Storage pattern:
+  - full conversation text in KV (`conv:*`)
+  - vector metadata in Vectorize
+- Ingestion supports JSON array/object conversation exports
+
+### v2 (`/api/v2/workspaces/:workspaceId/notebooks/*`)
+
+- Primary primitive: Knowledge Notebook (workspace scoped)
+- Storage pattern:
+  - notebooks/sources/chunks/jobs in D1
+  - vectors in Vectorize with notebook/source/chunk metadata pointers
+  - raw files in R2
+- Ingestion supports parser-typed jobs from source registration
+- Current parser coverage: `markdown`, `txt` (+ NDJSON thread-compatible ingest path)
+
+## Query Flow
+
+1. request enters notebook scoped route
+2. embedding generated via Workers AI
+3. vector search in Vectorize
+4. notebook scope enforced via metadata filter + app-level guard
+5. fallback to D1 chunk retrieval when vector/generation path degrades
+
+## Deletion Model
+
+- Notebook delete is soft-delete (`deleted_at`) in D1
+- Sources under notebook are soft-deleted together
+- This avoids irreversible data loss and enables async cleanup workflows
+
+## Design Constraints
+
+- Module size cap: < 400 lines per source module
+- Keep runtime dependency surface minimal (Hono only)
+- Prefer deterministic, source-grounded outputs with explicit citations
+- Keep OSS-safe boundaries: no secrets/PII artifacts in tracked files

ROADMAP.md

@@ -0,0 +1,32 @@
+# MindSpring Roadmap
+
+## Now
+
+- v1 production flows stable:
+  - conversation upload (`/api/uploads/*`)
+  - semantic search (`/api/search`)
+  - RAG chat (`/api/chat`)
+- v2 backend foundation live:
+  - notebooks CRUD/read lifecycle (workspace scoped)
+  - source registration + ingestion jobs
+  - job status polling
+  - notebook chat with citation fallback
+  - notebook search + chunk diagnostics
+- v2 parsers currently implemented:
+  - `markdown`
+  - `txt`
+  - NDJSON thread ingestion path for AEGIS-compatible writes
+
+## Next
+
+- Implement v2 artifact persistence (`POST /artifacts`) with `snapshot_hashes`
+- Implement `chat_export` parser under v2 pipeline
+- Add source-level invalidation/staleness markers for artifacts
+- Add migration bridge strategy for optional v1 -> v2 notebook wrapping
+
+## Later
+
+- Implement `url` parser
+- Implement `pdf` parser
+- Add notebook-first frontend routes once backend is fully stabilized
+- Add release automation and tagged version docs for v2 milestones

openapi.yaml

@@ -847,6 +847,61 @@ paths:
         '404':
           $ref: '#/components/responses/NotFound'
 
+  /api/v2/workspaces/{workspaceId}/notebooks/{notebookId}/search:
+    post:
+      summary: Notebook-scoped semantic search
+      operationId: notebookSearchV2
+      tags: [notebooks-v2]
+      x-auth-scope: read
+      parameters:
+        - $ref: '#/components/parameters/WorkspaceId'
+        - $ref: '#/components/parameters/NotebookId'
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [query]
+              properties:
+                query:
+                  type: string
+                limit:
+                  type: integer
+                  minimum: 1
+                  maximum: 50
+                threshold:
+                  type: number
+                  minimum: 0
+                  maximum: 1
+      responses:
+        '200':
+          description: Notebook scoped search results
+        '404':
+          $ref: '#/components/responses/NotFound'
+
+  /api/v2/workspaces/{workspaceId}/notebooks/{notebookId}/chunks:
+    get:
+      summary: Notebook chunk diagnostics
+      operationId: notebookChunksV2
+      tags: [notebooks-v2]
+      x-auth-scope: read
+      parameters:
+        - $ref: '#/components/parameters/WorkspaceId'
+        - $ref: '#/components/parameters/NotebookId'
+        - name: limit
+          in: query
+          schema:
+            type: integer
+            minimum: 1
+            maximum: 200
+            default: 50
+      responses:
+        '200':
+          description: Raw chunk diagnostics for notebook
+        '404':
+          $ref: '#/components/responses/NotFound'
+
   /api/v2/workspaces/{workspaceId}/notebooks/{notebookId}/artifacts:
     post:
       summary: Build studio artifact

src/lib/v2-store.ts

@@ -291,3 +291,40 @@ export async function getSource(
 
   return row ?? null
 }
+
+export interface NotebookChunkRecord {
+  id: string
+  notebook_id: string
+  source_id: string
+  content: string
+  chunk_hash: string
+  char_start: number
+  char_end: number
+  created_at: string
+  source_title: string
+}
+
+export async function listNotebookChunks(
+  env: Env,
+  workspaceId: string,
+  notebookId: string,
+  limit: number = 50
+): Promise<NotebookChunkRecord[]> {
+  const db = requireDb(env)
+  const rows = await db
+    .prepare(
+      `SELECT
+         c.id, c.notebook_id, c.source_id, c.content, c.chunk_hash, c.char_start, c.char_end, c.created_at,
+         s.title AS source_title
+       FROM chunks c
+       JOIN sources s ON s.id = c.source_id
+       JOIN notebooks n ON n.id = c.notebook_id
+       WHERE n.workspace_id = ? AND c.notebook_id = ? AND n.deleted_at IS NULL
+       ORDER BY c.created_at DESC
+       LIMIT ?`
+    )
+    .bind(workspaceId, notebookId, limit)
+    .all<NotebookChunkRecord>()
+
+  return rows.results
+}

src/routes/notebooks-v2-manage.ts

@@ -3,11 +3,14 @@ import type { Env } from '../lib/types'
 import {
   getNotebook,
   getSource,
+  listNotebookChunks,
   listNotebooks,
   listSources,
   patchNotebook,
   softDeleteNotebook,
 } from '../lib/v2-store'
+import { generateQueryEmbedding } from '../lib/embeddings'
+import { VectorStore } from '../lib/vectorize'
 
 const notebooksV2Manage = new Hono<{ Bindings: Env }>()
 
@@ -113,4 +116,57 @@ notebooksV2Manage.get('/:notebookId/sources/:sourceId', async (c) => {
   return c.json(source)
 })
 
+notebooksV2Manage.post('/:notebookId/search', async (c) => {
+  const workspaceId = requireParam(c.req.param('workspaceId'), 'workspaceId')
+  const notebookId = requireParam(c.req.param('notebookId'), 'notebookId')
+  const body = await c.req.json<{
+    query: string
+    limit?: number
+    threshold?: number
+  }>()
+
+  if (!body.query?.trim()) {
+    return c.json({ error: 'query is required' }, 400)
+  }
+
+  const notebook = await getNotebook(c.env, workspaceId, notebookId)
+  if (!notebook) return c.json({ error: 'notebook not found' }, 404)
+
+  const queryVector = await generateQueryEmbedding(body.query, c.env)
+  const store = new VectorStore(c.env)
+  const limit = Math.min(Math.max(body.limit ?? 10, 1), 50)
+  const threshold = body.threshold ?? 0
+
+  const results = await store.search(queryVector, limit, threshold, {
+    hydrateFullText: true,
+    notebookId,
+  })
+
+  return c.json({
+    query: body.query,
+    notebookId,
+    count: results.length,
+    results,
+  })
+})
+
+notebooksV2Manage.get('/:notebookId/chunks', async (c) => {
+  const workspaceId = requireParam(c.req.param('workspaceId'), 'workspaceId')
+  const notebookId = requireParam(c.req.param('notebookId'), 'notebookId')
+  const limit = Math.min(
+    Math.max(parseInt(c.req.query('limit') ?? '50', 10) || 50, 1),
+    200
+  )
+
+  const notebook = await getNotebook(c.env, workspaceId, notebookId)
+  if (!notebook) return c.json({ error: 'notebook not found' }, 404)
+
+  const chunks = await listNotebookChunks(c.env, workspaceId, notebookId, limit)
+  return c.json({
+    notebookId,
+    count: chunks.length,
+    chunks,
+  })
+})
+
 export { notebooksV2Manage }