feat: continue restart plan

Author: gustavotoyota

new-deepnotes/PLAN_PROGRESS.md

@@ -127,6 +127,8 @@ describe("api-worker", () => {
     ],
     ["POST", "/api/pages/aaaaaaaaaaaaaaaaaaaaa/move"],
     ["POST", "/api/pages/aaaaaaaaaaaaaaaaaaaaa/bump"],
+    ["GET", "/api/pages/aaaaaaaaaaaaaaaaaaaaa/collab-updates"],
+    ["POST", "/api/pages/aaaaaaaaaaaaaaaaaaaaa/collab-updates"],
     ["POST", "/api/pages/aaaaaaaaaaaaaaaaaaaaa/backlinks"],
     [
       "DELETE",

new-deepnotes/apps/api-worker/src/index.test.ts

@@ -6,6 +6,7 @@ import {
   groupPagesListQuerySchema,
   pageBacklinkCreateRequestSchema,
   pageBumpRequestSchema,
+  pageCollabUpdatesAppendRequestSchema,
   pageMoveRequestSchema,
   pageIdPathSchema,
   pageSnapshotCreateResponseSchema,
@@ -1577,6 +1578,130 @@ app.post("/api/pages/:pageId/bump", async (c) => {
   }
 });
 
+app.get("/api/pages/:pageId/collab-updates", async (c) => {
+  const sessionEnv = getSessionEnv(c.env);
+  if (sessionEnv == null) {
+    return c.json(serviceUnavailableBody, 503);
+  }
+  const hyper = c.env.HYPERDRIVE;
+  if (hyper == null) {
+    return c.json(
+      {
+        code: "SERVICE_UNAVAILABLE" as const,
+        message: "HYPERDRIVE binding is not configured.",
+      },
+      503,
+    );
+  }
+
+  const pParams = pageIdPathSchema.safeParse({ pageId: c.req.param("pageId") });
+  if (!pParams.success) {
+    return c.json(
+      { code: "VALIDATION_ERROR", message: pParams.error.message },
+      400,
+    );
+  }
+
+  const db = getDbForConnectionString(hyper.connectionString);
+  const cookieHeader = c.req.header("Cookie");
+
+  try {
+    const { performGetPageCollabUpdates } = await import("@deepnotes/session");
+    const out = await performGetPageCollabUpdates({
+      db,
+      env: sessionEnv,
+      accessCookie: readCookieHeader(cookieHeader, "accessToken"),
+      pageId: pParams.data.pageId,
+    });
+    return c.json(
+      {
+        lastIndex: out.lastIndex,
+        updates: out.updates.map((u) => ({
+          index: u.index,
+          encryptedData: u.encryptedData.toString("base64"),
+        })),
+      },
+      200,
+    );
+  } catch (e) {
+    const { SessionError } = await import("@deepnotes/session");
+    if (e instanceof SessionError) {
+      return c.json(
+        { code: e.code, message: e.message },
+        e.status as ContentfulStatusCode,
+      );
+    }
+    throw e;
+  }
+});
+
+app.post("/api/pages/:pageId/collab-updates", async (c) => {
+  const sessionEnv = getSessionEnv(c.env);
+  if (sessionEnv == null) {
+    return c.json(serviceUnavailableBody, 503);
+  }
+  const hyper = c.env.HYPERDRIVE;
+  if (hyper == null) {
+    return c.json(
+      {
+        code: "SERVICE_UNAVAILABLE" as const,
+        message: "HYPERDRIVE binding is not configured.",
+      },
+      503,
+    );
+  }
+
+  let bodyJson: unknown;
+  try {
+    bodyJson = await c.req.json();
+  } catch {
+    return c.json({ code: "BAD_REQUEST", message: "Expected JSON body." }, 400);
+  }
+
+  const pParams = pageIdPathSchema.safeParse({ pageId: c.req.param("pageId") });
+  if (!pParams.success) {
+    return c.json(
+      { code: "VALIDATION_ERROR", message: pParams.error.message },
+      400,
+    );
+  }
+  const parsed = pageCollabUpdatesAppendRequestSchema.safeParse(bodyJson);
+  if (!parsed.success) {
+    return c.json(
+      {
+        code: "VALIDATION_ERROR",
+        message: parsed.error.flatten().formErrors.join("; "),
+      },
+      400,
+    );
+  }
+
+  const db = getDbForConnectionString(hyper.connectionString);
+  const cookieHeader = c.req.header("Cookie");
+
+  try {
+    const { performAppendPageCollabUpdates } = await import("@deepnotes/session");
+    await performAppendPageCollabUpdates({
+      db,
+      env: sessionEnv,
+      accessCookie: readCookieHeader(cookieHeader, "accessToken"),
+      pageId: pParams.data.pageId,
+      expectedLastIndex: parsed.data.expectedLastIndex,
+      updates: parsed.data.updates,
+    });
+    return c.body(null, 204);
+  } catch (e) {
+    const { SessionError } = await import("@deepnotes/session");
+    if (e instanceof SessionError) {
+      return c.json(
+        { code: e.code, message: e.message },
+        e.status as ContentfulStatusCode,
+      );
+    }
+    throw e;
+  }
+});
+
 app.post("/api/pages/:pageId/backlinks", async (c) => {
   const sessionEnv = getSessionEnv(c.env);
   if (sessionEnv == null) {

new-deepnotes/apps/api-worker/src/index.ts

@@ -79,6 +79,13 @@ Working checklist for Phase 0 of [docs/RESTART_PLAN.md](../../docs/RESTART_PLAN.
 | `pages.deletion.restore` | `POST /api/pages/:pageId/restore` (**implemented** — `performPageRestore`) |
 | `pages.deletion.deletePermanently` | `POST /api/pages/:pageId/purge` (**implemented** — `performPagePurge`; `num_free_pages` +1 when `pages.free` and user not Pro) |
 
+### Collab bootstrap (new; not legacy tRPC)
+
+| Capability | New surface | Notes |
+|------------|-------------|--------|
+| Load encrypted Yjs update chain from DB | `GET /api/pages/:pageId/collab-updates` (**implemented** — `performGetPageCollabUpdates`; `viewGroupPages`; Postgres `page_updates` only, no Redis cache) | Replaces initial `ALL_UPDATES_UNMERGED`-style payload for SPA bootstrap; binary collab WebSocket is still [Phase 3 — realtime/collab](../PLAN_PROGRESS.md#not-started-phase-3--realtime--collab-only). |
+| Append updates (optimistic concurrency) | `POST /api/pages/:pageId/collab-updates` (**implemented** — `performAppendPageCollabUpdates`; `editGroupPages`; body `expectedLastIndex` + `updates[]`) | **409** when `expectedLastIndex` is stale. |
+
 ## Legacy app-server WebSocket → target
 
 | Legacy handler | New surface | Notes |

new-deepnotes/docs/TRPC_REST_MAP.md

@@ -46,6 +46,8 @@ export {
   groupUserIdPathSchema,
   pageBacklinkCreateRequestSchema,
   pageBumpRequestSchema,
+  pageCollabUpdatesAppendRequestSchema,
+  pageCollabUpdatesGetResponseSchema,
   pageMoveRequestSchema,
   pageIdPathSchema,
   pageSnapshotCreateResponseSchema,

new-deepnotes/packages/api/src/index.ts

@@ -105,6 +105,12 @@ describe("getOpenApiDocument", () => {
     ).toBeDefined();
     expect(doc.paths?.["/api/pages/{pageId}/move"]?.post).toBeDefined();
     expect(doc.paths?.["/api/pages/{pageId}/bump"]?.post).toBeDefined();
+    expect(
+      doc.paths?.["/api/pages/{pageId}/collab-updates"]?.get,
+    ).toBeDefined();
+    expect(
+      doc.paths?.["/api/pages/{pageId}/collab-updates"]?.post,
+    ).toBeDefined();
     expect(
       doc.paths?.["/api/pages/{pageId}/backlinks"]?.post,
     ).toBeDefined();

new-deepnotes/packages/api/src/openapi.test.ts

@@ -38,6 +38,8 @@ import {
   groupUserIdPathSchema,
   pageBacklinkCreateRequestSchema,
   pageBumpRequestSchema,
+  pageCollabUpdatesAppendRequestSchema,
+  pageCollabUpdatesGetResponseSchema,
   pageMoveRequestSchema,
   pageIdPathSchema,
   pageSnapshotCreateResponseSchema,
@@ -1202,6 +1204,60 @@ registry.registerPath({
   },
 });
 
+registry.registerPath({
+  method: "get",
+  path: "/api/pages/{pageId}/collab-updates",
+  summary: "List encrypted Yjs page updates (Postgres)",
+  description:
+    "Bootstrap for the editor: returns all `page_updates` rows for the page, ordered by `index`. Does not use legacy Redis collab cache — Postgres only. Full duplex collab remains a separate WebSocket track (Phase 3).",
+  request: { params: pageIdPathSchema },
+  responses: {
+    200: {
+      description: "Current ciphertext chain.",
+      content: {
+        "application/json": { schema: pageCollabUpdatesGetResponseSchema },
+      },
+    },
+    401: sessionUnauthorized401,
+    403: sessionForbidden403,
+    404: sessionNotFound404,
+    503: sessionServiceUnavailable503,
+  },
+});
+
+registry.registerPath({
+  method: "post",
+  path: "/api/pages/{pageId}/collab-updates",
+  summary: "Append page updates (optimistic concurrency)",
+  description:
+    "Appends ciphertext rows to `page_updates`. `expectedLastIndex` must match the current max index (or null when empty). **409** when another writer advanced the chain — client should re-GET and retry.",
+  request: {
+    params: pageIdPathSchema,
+    body: {
+      content: {
+        "application/json": {
+          schema: pageCollabUpdatesAppendRequestSchema,
+        },
+      },
+    },
+  },
+  responses: {
+    204: { description: "Updates persisted." },
+    400: {
+      description: "Bad index sequence or wrong `expectedLastIndex` for an empty page.",
+      content: { "application/json": { schema: sessionErrorResponseSchema } },
+    },
+    401: sessionUnauthorized401,
+    403: sessionForbidden403,
+    404: sessionNotFound404,
+    409: {
+      description: "Stale `expectedLastIndex` (concurrent append).",
+      content: { "application/json": { schema: sessionErrorResponseSchema } },
+    },
+    503: sessionServiceUnavailable503,
+  },
+});
+
 registry.registerPath({
   method: "post",
   path: "/api/pages/{pageId}/backlinks",

new-deepnotes/packages/api/src/openapi.ts

@@ -350,3 +350,56 @@ export const pageSnapshotLoadResponseSchema = z
       .openapi({ format: "byte", description: "Base64 ciphertext." }),
   })
   .openapi("PageSnapshotLoadResponse");
+
+const pageCollabUpdateItemInputSchema = z
+  .object({
+    index: z
+      .number()
+      .int()
+      .nonnegative()
+      .openapi({
+        description:
+          "Monotonic index (legacy collab / `page_updates.index`, often Yjs clock).",
+      }),
+    encryptedData: byteB64,
+  })
+  .openapi("PageCollabUpdateItemInput");
+
+export const pageCollabUpdatesAppendRequestSchema = z
+  .object({
+    expectedLastIndex: z
+      .number()
+      .int()
+      .nonnegative()
+      .nullable()
+      .openapi({
+        description:
+          "Must match `lastIndex` from GET (`null` when the page has no updates yet).",
+      }),
+    updates: z.array(pageCollabUpdateItemInputSchema).min(1),
+  })
+  .openapi("PageCollabUpdatesAppendRequest");
+
+export const pageCollabUpdatesGetResponseSchema = z
+  .object({
+    lastIndex: z
+      .number()
+      .int()
+      .nonnegative()
+      .nullable()
+      .openapi({
+        description: "Max `index` in the database, or null if there are no rows.",
+      }),
+    updates: z.array(
+      z.object({
+        index: z.number().int().nonnegative(),
+        encryptedData: z
+          .string()
+          .openapi({
+            format: "byte",
+            description: "Base64 ciphertext (`page_updates.encrypted_data`).",
+          }),
+      }),
+    ),
+  })
+  .openapi("PageCollabUpdatesGetResponse");

new-deepnotes/packages/api/src/schemas/pages-groups.ts

@@ -88,6 +88,8 @@ import {
   performPageSnapshotLoad,
   performPageSnapshotSave,
   performPageSoftDelete,
+  performAppendPageCollabUpdates,
+  performGetPageCollabUpdates,
 } from "./index.js";
 import {
   performCreatePage,
@@ -1374,6 +1376,79 @@ describe.skipIf(resolveTemplateContext() == null)(
             groupId: "nononononononononono1",
           }),
         ).rejects.toMatchObject({ status: 404, code: "NOT_FOUND" });
+
+        const empty = await performGetPageCollabUpdates({
+          db,
+          env,
+          accessCookie: access,
+          pageId: reg.pageId,
+        });
+        expect(empty.lastIndex).toBeNull();
+        expect(empty.updates).toEqual([]);
+
+        const b0 = rand32();
+        await performAppendPageCollabUpdates({
+          db,
+          env,
+          accessCookie: access,
+          pageId: reg.pageId,
+          expectedLastIndex: null,
+          updates: [{ index: 0, encryptedData: b0 }],
+        });
+
+        const one = await performGetPageCollabUpdates({
+          db,
+          env,
+          accessCookie: access,
+          pageId: reg.pageId,
+        });
+        expect(one.lastIndex).toBe(0);
+        expect(one.updates).toHaveLength(1);
+        expect(one.updates[0]!.index).toBe(0);
+        expect(one.updates[0]!.encryptedData.equals(Buffer.from(b0))).toBe(
+          true,
+        );
+
+        const b1 = rand32();
+        await performAppendPageCollabUpdates({
+          db,
+          env,
+          accessCookie: access,
+          pageId: reg.pageId,
+          expectedLastIndex: 0,
+          updates: [{ index: 1, encryptedData: b1 }],
+        });
+
+        const two = await performGetPageCollabUpdates({
+          db,
+          env,
+          accessCookie: access,
+          pageId: reg.pageId,
+        });
+        expect(two.lastIndex).toBe(1);
+        expect(two.updates).toHaveLength(2);
+
+        await expect(
+          performAppendPageCollabUpdates({
+            db,
+            env,
+            accessCookie: access,
+            pageId: reg.pageId,
+            expectedLastIndex: 0,
+            updates: [{ index: 2, encryptedData: rand32() }],
+          }),
+        ).rejects.toMatchObject({ status: 409, code: "CONFLICT" });
+
+        await expect(
+          performAppendPageCollabUpdates({
+            db,
+            env,
+            accessCookie: access,
+            pageId: reg.pageId,
+            expectedLastIndex: 1,
+            updates: [{ index: 3, encryptedData: rand32() }],
+          }),
+        ).rejects.toMatchObject({ status: 400, code: "BAD_REQUEST" });
       } finally {
         await client.end({ timeout: 5 });
         const admin2 = postgres(ctx.adminUrl, { max: 1 });