diff --git a/packages/vinext/src/cloudflare/populate-kv.ts b/packages/vinext/src/cloudflare/populate-kv.ts new file mode 100644 index 00000000..bac7c2ca --- /dev/null +++ b/packages/vinext/src/cloudflare/populate-kv.ts @@ -0,0 +1,330 @@ +/** + * Deploy-time KV population for Cloudflare Workers. + * + * Uploads pre-rendered HTML and RSC data to Workers KV during deployment + * so first requests are served from cache instead of rendered on the fly. + * + * Key construction and tag generation match the runtime functions in + * entries/app-rsc-entry.ts exactly — see appPageCacheKey and buildPageTags. + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fnv1a64 } from "../utils/hash.js"; +import { getOutputPath, getRscOutputPath } from "../build/prerender.js"; + +/** Key prefix for cache entries. Matches kv-cache-handler.ts ENTRY_PREFIX. */ +export const ENTRY_PREFIX = "cache:"; + +/** Default KV TTL in seconds (30 days). Matches KVCacheHandler default. */ +export const KV_TTL_SECONDS = 30 * 24 * 3600; + +// ─── Cache key construction ───────────────────────────────────────────── + +/** + * Construct an ISR cache key for an App Router page, matching the runtime + * `__isrCacheKey(pathname, suffix)` in the generated RSC entry exactly. + * + * Format: `app:[buildId:]:` + * Long keys (>200 chars): `app:[buildId:]__hash::` + */ +export function appPageCacheKey( + pathname: string, + suffix: "html" | "rsc" | "route", + buildId?: string, +): string { + const normalized = pathname === "/" ? "/" : pathname.replace(/\/$/, ""); + const prefix = buildId ? `app:${buildId}` : "app"; + const key = `${prefix}:${normalized}:${suffix}`; + if (key.length <= 200) return key; + return `${prefix}:__hash:${fnv1a64(normalized)}:${suffix}`; +} + +// ─── Tag construction ─────────────────────────────────────────────────── + +/** + * Build cache tags for a page, matching the runtime `__pageCacheTags(pathname)` + * in the generated RSC entry exactly. + * + * Produces: pathname, _N_T_ prefixed pathname, root layout tag, intermediate + * layout tags for each segment, and a leaf page tag. + */ +export function buildPageTags(pathname: string): string[] { + const tags = [pathname, `_N_T_${pathname}`]; + tags.push("_N_T_/layout"); + const segments = pathname.split("/"); + let built = ""; + for (let i = 1; i < segments.length; i++) { + if (segments[i]) { + built += "/" + segments[i]; + tags.push(`_N_T_${built}/layout`); + } + } + tags.push(`_N_T_${built}/page`); + return tags; +} + +// ─── Entry serialization ──────────────────────────────────────────────── + +interface AppPageValue { + kind: "APP_PAGE"; + html: string; + rscData?: Buffer; + headers?: Record; + postponed?: string; + status: number; +} + +/** + * Build a serialized KVCacheEntry JSON string matching the format that + * KVCacheHandler.get() expects at runtime. + * + * rscData (Buffer) is base64-encoded. Other fields pass through as-is. + */ +export function buildKVCacheEntryJSON( + value: AppPageValue, + tags: string[], + revalidateSeconds: number | false, +): string { + const now = Date.now(); + const revalidateAt = + typeof revalidateSeconds === "number" && revalidateSeconds > 0 + ? now + revalidateSeconds * 1000 + : null; + + const serializedValue = { + kind: value.kind, + html: value.html, + rscData: value.rscData ? value.rscData.toString("base64") : undefined, + headers: value.headers, + postponed: value.postponed, + status: value.status, + }; + + return JSON.stringify({ + value: serializedValue, + tags, + lastModified: now, + revalidateAt, + }); +} + +// ─── Route entry construction ─────────────────────────────────────────── + +export interface KVBulkPair { + key: string; + value: string; + expiration_ttl?: number; +} + +/** + * Build KV bulk API pairs for a single pre-rendered route. + * + * Produces up to 2 entries: + * - `:html` entry with the rendered HTML (always) + * - `:rsc` entry with the RSC payload (when rscBuffer is provided) + */ +export function buildRouteEntries( + pathname: string, + html: string, + rscBuffer: Buffer | null, + revalidate: number | false, + buildId: string, + appPrefix?: string, +): KVBulkPair[] { + const kvPrefix = appPrefix ? `${appPrefix}:${ENTRY_PREFIX}` : ENTRY_PREFIX; + const tags = buildPageTags(pathname); + const expiration_ttl = + typeof revalidate === "number" && revalidate > 0 ? KV_TTL_SECONDS : undefined; + + const pairs: KVBulkPair[] = [ + { + key: `${kvPrefix}${appPageCacheKey(pathname, "html", buildId)}`, + value: buildKVCacheEntryJSON({ kind: "APP_PAGE", html, status: 200 }, tags, revalidate), + expiration_ttl, + }, + ]; + + if (rscBuffer) { + pairs.push({ + key: `${kvPrefix}${appPageCacheKey(pathname, "rsc", buildId)}`, + value: buildKVCacheEntryJSON( + { kind: "APP_PAGE", html: "", rscData: rscBuffer, status: 200 }, + tags, + revalidate, + ), + expiration_ttl, + }); + } + + return pairs; +} + +// ─── Bulk upload ──────────────────────────────────────────────────────── + +const MAX_BATCH_COUNT = 10_000; +const MAX_BATCH_BYTES = 95 * 1024 * 1024; // 95 MB (5 MB headroom from 100 MB limit) + +/** + * Upload KV entries via the Cloudflare REST bulk API. + * Batches by entry count (max 10,000) and payload byte size (max ~95 MB). + */ +export async function uploadBulkToKV( + pairs: KVBulkPair[], + namespaceId: string, + accountId: string, + apiToken: string, +): Promise { + const batches: KVBulkPair[][] = []; + let currentBatch: KVBulkPair[] = []; + let currentBytes = 0; + + for (const pair of pairs) { + const pairBytes = Buffer.byteLength(pair.key) + Buffer.byteLength(pair.value); + + if ( + currentBatch.length >= MAX_BATCH_COUNT || + (currentBatch.length > 0 && currentBytes + pairBytes > MAX_BATCH_BYTES) + ) { + batches.push(currentBatch); + currentBatch = []; + currentBytes = 0; + } + + currentBatch.push(pair); + currentBytes += pairBytes; + } + + if (currentBatch.length > 0) { + batches.push(currentBatch); + } + + const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/bulk`; + const headers = { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + }; + + for (let i = 0; i < batches.length; i++) { + const response = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(batches[i]), + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error( + `KV bulk upload failed (batch ${i + 1}/${batches.length}): ${response.status} — ${text}`, + ); + } + } +} + +// ─── Orchestrator ─────────────────────────────────────────────────────── + +interface PrerenderManifestRoute { + route: string; + status: string; + revalidate?: number | false; + path?: string; + /** Router type — only "app" routes are seeded (Pages Router has a different key/value schema). */ + router?: "app" | "pages"; +} + +interface PrerenderManifest { + buildId?: string; + trailingSlash?: boolean; + routes: PrerenderManifestRoute[]; +} + +export interface PopulateKVOptions { + root: string; + accountId: string; + namespaceId: string; + apiToken: string; + appPrefix?: string; +} + +export interface PopulateKVResult { + routesProcessed: number; + entriesUploaded: number; + skipped?: string; + durationMs: number; +} + +/** + * Populate Workers KV with pre-rendered pages from the build output. + * + * Reads the prerender manifest and HTML/RSC files, constructs cache entries + * matching the runtime format exactly, and uploads via the Cloudflare bulk API. + */ +export async function populateKV(options: PopulateKVOptions): Promise { + const start = performance.now(); + const serverDir = path.join(options.root, "dist", "server"); + const manifestPath = path.join(serverDir, "vinext-prerender.json"); + + if (!fs.existsSync(manifestPath)) { + return { + routesProcessed: 0, + entriesUploaded: 0, + skipped: "no prerender manifest", + durationMs: 0, + }; + } + + const manifest: PrerenderManifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); + + if (!manifest.buildId) { + return { + routesProcessed: 0, + entriesUploaded: 0, + skipped: "manifest missing buildId", + durationMs: 0, + }; + } + + const prerenderDir = path.join(serverDir, "prerendered-routes"); + const trailingSlash = manifest.trailingSlash ?? false; + const allPairs: KVBulkPair[] = []; + let routesProcessed = 0; + + for (const route of manifest.routes) { + if (route.status !== "rendered") continue; + // Only seed App Router routes — Pages Router uses different keys (pages:...) + // and a different value shape (kind: "PAGES"). The router field is set by + // PR #653's enriched manifest; routes without it are skipped defensively. + if (route.router !== "app") continue; + + const pathname = route.path ?? route.route; + const htmlFile = path.join(prerenderDir, getOutputPath(pathname, trailingSlash)); + + if (!fs.existsSync(htmlFile)) continue; + + const html = fs.readFileSync(htmlFile, "utf-8"); + const rscFile = path.join(prerenderDir, getRscOutputPath(pathname)); + const rscBuffer = fs.existsSync(rscFile) ? fs.readFileSync(rscFile) : null; + + const pairs = buildRouteEntries( + pathname, + html, + rscBuffer, + route.revalidate ?? false, + manifest.buildId, + options.appPrefix, + ); + + allPairs.push(...pairs); + routesProcessed++; + } + + if (allPairs.length > 0) { + await uploadBulkToKV(allPairs, options.namespaceId, options.accountId, options.apiToken); + } + + return { + routesProcessed, + entriesUploaded: allPairs.length, + durationMs: performance.now() - start, + }; +} diff --git a/packages/vinext/src/cloudflare/tpr.ts b/packages/vinext/src/cloudflare/tpr.ts index 5dcfc130..4b4bca97 100644 --- a/packages/vinext/src/cloudflare/tpr.ts +++ b/packages/vinext/src/cloudflare/tpr.ts @@ -24,6 +24,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { spawn, type ChildProcess } from "node:child_process"; +import { buildRouteEntries, uploadBulkToKV, type KVBulkPair } from "./populate-kv.js"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -359,7 +360,7 @@ async function resolveZoneId(domain: string, apiToken: string): Promise { +export async function resolveAccountId(apiToken: string): Promise { const response = await fetch("https://api.cloudflare.com/client/v4/accounts?per_page=1", { headers: { Authorization: `Bearer ${apiToken}`, @@ -662,9 +663,10 @@ async function waitForServer(port: number, timeoutMs: number): Promise { // ─── KV Upload ─────────────────────────────────────────────────────────────── /** - * Upload pre-rendered pages to KV using the Cloudflare REST API. - * Writes in the same KVCacheEntry format that KVCacheHandler reads - * at runtime, so ISR serves these entries without any code changes. + * Upload pre-rendered pages to KV using shared populate-kv helpers. + * + * Uses the same key format, tag construction, and TTL as the deploy-time + * KV population step and the runtime KVCacheHandler. */ async function uploadToKV( entries: Map, @@ -672,76 +674,23 @@ async function uploadToKV( accountId: string, apiToken: string, defaultRevalidateSeconds: number, + buildId?: string, ): Promise { - const now = Date.now(); - - // Build the bulk write payload - const pairs: Array<{ - key: string; - value: string; - expiration_ttl?: number; - }> = []; + const allPairs: KVBulkPair[] = []; for (const [routePath, result] of entries) { - // Determine revalidation window — use the page's revalidate header - // if present, otherwise fall back to the default const revalidateHeader = result.headers["x-vinext-revalidate"]; const revalidateSeconds = revalidateHeader && !isNaN(Number(revalidateHeader)) ? Number(revalidateHeader) : defaultRevalidateSeconds; - const revalidateAt = revalidateSeconds > 0 ? now + revalidateSeconds * 1000 : null; - - // KV TTL: 10x the revalidation period, clamped to [60s, 30d] - // (matches the logic in KVCacheHandler.set) - const kvTtl = - revalidateSeconds > 0 - ? Math.max(Math.min(revalidateSeconds * 10, 30 * 24 * 3600), 60) - : 24 * 3600; // 24h fallback if no revalidation - - const entry = { - value: { - kind: "APP_PAGE" as const, - html: result.html, - headers: result.headers, - status: result.status, - }, - tags: [] as string[], - lastModified: now, - revalidateAt, - }; - - pairs.push({ - key: `cache:${routePath}`, - value: JSON.stringify(entry), - expiration_ttl: kvTtl, - }); + // TPR only captures HTML (no RSC data from HTTP fetch) + const pairs = buildRouteEntries(routePath, result.html, null, revalidateSeconds, buildId ?? ""); + allPairs.push(...pairs); } - // Upload in batches (KV bulk API accepts up to 10,000 per request) - const BATCH_SIZE = 10_000; - for (let i = 0; i < pairs.length; i += BATCH_SIZE) { - const batch = pairs.slice(i, i + BATCH_SIZE); - const response = await fetch( - `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/bulk`, - { - method: "PUT", - headers: { - Authorization: `Bearer ${apiToken}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(batch), - }, - ); - - if (!response.ok) { - const text = await response.text(); - throw new Error( - `KV bulk upload failed (batch ${Math.floor(i / BATCH_SIZE) + 1}): ${response.status} — ${text}`, - ); - } - } + await uploadBulkToKV(allPairs, namespaceId, accountId, apiToken); } // ─── Main Entry ────────────────────────────────────────────────────────────── @@ -856,6 +805,18 @@ export async function runTPR(options: TPROptions): Promise { } // ── 10. Upload to KV ────────────────────────────────────────── + // Resolve buildId from prerender manifest (written during build) + let buildId: string | undefined; + try { + const manifestPath = path.join(root, "dist", "server", "vinext-prerender.json"); + if (fs.existsSync(manifestPath)) { + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); + buildId = manifest.buildId; + } + } catch { + // Best-effort — proceed without buildId + } + try { await uploadToKV( rendered, @@ -863,6 +824,7 @@ export async function runTPR(options: TPROptions): Promise { accountId, apiToken, DEFAULT_REVALIDATE_SECONDS, + buildId, ); } catch (err) { return skip(`KV upload failed: ${err instanceof Error ? err.message : String(err)}`); diff --git a/packages/vinext/src/deploy.ts b/packages/vinext/src/deploy.ts index 1d2efaca..af1c2281 100644 --- a/packages/vinext/src/deploy.ts +++ b/packages/vinext/src/deploy.ts @@ -27,7 +27,8 @@ import { findInNodeModules as _findInNodeModules, } from "./utils/project.js"; import { getReactUpgradeDeps } from "./init.js"; -import { runTPR } from "./cloudflare/tpr.js"; +import { parseWranglerConfig, resolveAccountId, runTPR } from "./cloudflare/tpr.js"; +import { populateKV } from "./cloudflare/populate-kv.js"; import { runPrerender } from "./build/run-prerender.js"; import { loadDotenv } from "./config/dotenv.js"; import { loadNextConfig, resolveNextConfig } from "./config/next-config.js"; @@ -57,6 +58,8 @@ export interface DeployOptions { tprLimit?: number; /** TPR: analytics lookback window in hours (default: 24) */ tprWindow?: number; + /** Disable automatic KV population with pre-rendered pages */ + noPopulateKv?: boolean; } // ─── CLI arg parsing (uses Node.js util.parseArgs) ────────────────────────── @@ -74,6 +77,7 @@ const deployArgOptions = { "tpr-coverage": { type: "string" }, "tpr-limit": { type: "string" }, "tpr-window": { type: "string" }, + "no-populate-kv": { type: "boolean", default: false }, } as const; export function parseDeployArgs(args: string[]) { @@ -101,6 +105,7 @@ export function parseDeployArgs(args: string[]) { tprCoverage: parseIntArg("tpr-coverage", values["tpr-coverage"]), tprLimit: parseIntArg("tpr-limit", values["tpr-limit"]), tprWindow: parseIntArg("tpr-window", values["tpr-window"]), + noPopulateKv: values["no-populate-kv"], }; } @@ -1356,6 +1361,34 @@ export async function deploy(options: DeployOptions): Promise { } } + // Step 6c: Populate KV with pre-rendered pages + if (!options.noPopulateKv) { + const apiToken = process.env.CLOUDFLARE_API_TOKEN; + const wranglerConfig = parseWranglerConfig(root); + const kvNamespaceId = wranglerConfig?.kvNamespaceId; + + if (apiToken && kvNamespaceId) { + const accountId = wranglerConfig?.accountId ?? (await resolveAccountId(apiToken)); + + if (accountId) { + const kvResult = await populateKV({ + root, + accountId, + namespaceId: kvNamespaceId, + apiToken, + }); + + if (kvResult.skipped) { + console.log(`\n KV populate: Skipped (${kvResult.skipped})`); + } else if (kvResult.entriesUploaded > 0) { + console.log( + `\n KV populate: ${kvResult.routesProcessed} routes → ${kvResult.entriesUploaded} entries (${(kvResult.durationMs / 1000).toFixed(1)}s)`, + ); + } + } + } + } + // Step 7: Deploy via wrangler const url = runWranglerDeploy(root, { preview: options.preview ?? false, diff --git a/tests/populate-kv.test.ts b/tests/populate-kv.test.ts new file mode 100644 index 00000000..0939b291 --- /dev/null +++ b/tests/populate-kv.test.ts @@ -0,0 +1,660 @@ +/** + * Tests for deploy-time KV population. + * + * Verifies key construction, tag generation, entry serialization, + * bulk upload batching, and end-to-end populate flow. + * + * Key parity tests ensure the deploy-time module produces keys and tags + * identical to the runtime functions in entries/app-rsc-entry.ts. + */ + +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, it, expect, vi, beforeEach } from "vite-plus/test"; +import { fnv1a64 } from "../packages/vinext/src/utils/hash.js"; +import { + appPageCacheKey, + buildPageTags, + buildKVCacheEntryJSON, + buildRouteEntries, + uploadBulkToKV, + populateKV, + ENTRY_PREFIX, + KV_TTL_SECONDS, + type KVBulkPair, +} from "../packages/vinext/src/cloudflare/populate-kv.js"; + +/** Extract and parse the JSON body from a mocked fetch RequestInit. */ +function parseFetchBody(init: RequestInit | undefined): KVBulkPair[] { + if (typeof init?.body !== "string") throw new Error("expected string body in fetch mock"); + return JSON.parse(init.body); +} + +// ─── appPageCacheKey ──────────────────────────────────────────────────── + +describe("appPageCacheKey", () => { + it("produces correct key for root path with buildId", () => { + expect(appPageCacheKey("/", "html", "abc123")).toBe("app:abc123:/:html"); + }); + + it("produces correct key for nested path with buildId", () => { + expect(appPageCacheKey("/blog/hello", "rsc", "abc123")).toBe("app:abc123:/blog/hello:rsc"); + }); + + it("strips trailing slash from non-root paths", () => { + expect(appPageCacheKey("/blog/hello/", "html", "abc123")).toBe("app:abc123:/blog/hello:html"); + }); + + it("preserves root / without stripping", () => { + expect(appPageCacheKey("/", "rsc", "build1")).toBe("app:build1:/:rsc"); + }); + + it("produces key without buildId when omitted", () => { + expect(appPageCacheKey("/about", "html")).toBe("app:/about:html"); + }); + + it("supports route suffix", () => { + expect(appPageCacheKey("/api/data", "route", "b1")).toBe("app:b1:/api/data:route"); + }); + + it("hashes long pathnames that exceed 200 char key limit", () => { + const longPath = "/" + "a".repeat(250); + const key = appPageCacheKey(longPath, "html", "b1"); + + // Key must use __hash: format + expect(key).toMatch(/^app:b1:__hash:.+:html$/); + // Must contain fnv1a64 hash of the normalized pathname + expect(key).toContain(fnv1a64(longPath)); + // Must be under 200 chars + expect(key.length).toBeLessThanOrEqual(200); + }); + + it("produces deterministic hash for the same long pathname", () => { + const longPath = "/" + "x".repeat(300); + const key1 = appPageCacheKey(longPath, "html", "b1"); + const key2 = appPageCacheKey(longPath, "html", "b1"); + expect(key1).toBe(key2); + }); + + it("produces different keys for html and rsc suffixes on same path", () => { + const htmlKey = appPageCacheKey("/page", "html", "b1"); + const rscKey = appPageCacheKey("/page", "rsc", "b1"); + expect(htmlKey).not.toBe(rscKey); + expect(htmlKey).toBe("app:b1:/page:html"); + expect(rscKey).toBe("app:b1:/page:rsc"); + }); +}); + +// ─── buildPageTags ────────────────────────────────────────────────────── + +describe("buildPageTags", () => { + it("produces correct tags for root path", () => { + // Matches __pageCacheTags("/") from app-rsc-entry.ts + const tags = buildPageTags("/"); + expect(tags).toContain("/"); + expect(tags).toContain("_N_T_/"); + expect(tags).toContain("_N_T_/layout"); + // Root has no intermediate segments, leaf page tag is _N_T_/page + expect(tags).toContain("_N_T_/page"); + }); + + it("produces correct tags for single-segment path", () => { + const tags = buildPageTags("/about"); + expect(tags).toEqual([ + "/about", + "_N_T_/about", + "_N_T_/layout", + "_N_T_/about/layout", + "_N_T_/about/page", + ]); + }); + + it("produces correct tags for nested path", () => { + const tags = buildPageTags("/blog/hello"); + expect(tags).toEqual([ + "/blog/hello", + "_N_T_/blog/hello", + "_N_T_/layout", + "_N_T_/blog/layout", + "_N_T_/blog/hello/layout", + "_N_T_/blog/hello/page", + ]); + }); + + it("produces correct tags for deeply nested path", () => { + const tags = buildPageTags("/a/b/c/d"); + expect(tags).toEqual([ + "/a/b/c/d", + "_N_T_/a/b/c/d", + "_N_T_/layout", + "_N_T_/a/layout", + "_N_T_/a/b/layout", + "_N_T_/a/b/c/layout", + "_N_T_/a/b/c/d/layout", + "_N_T_/a/b/c/d/page", + ]); + }); +}); + +// ─── buildKVCacheEntryJSON ────────────────────────────────────────────── + +describe("buildKVCacheEntryJSON", () => { + it("produces valid KVCacheEntry shape for HTML entry", () => { + const json = buildKVCacheEntryJSON( + { kind: "APP_PAGE", html: "

Hello

", status: 200 }, + ["/about"], + 60, + ); + const entry = JSON.parse(json); + + expect(entry.value).toEqual({ + kind: "APP_PAGE", + html: "

Hello

", + rscData: undefined, + headers: undefined, + postponed: undefined, + status: 200, + }); + expect(entry.tags).toEqual(["/about"]); + expect(typeof entry.lastModified).toBe("number"); + expect(entry.lastModified).toBeGreaterThan(0); + }); + + it("sets revalidateAt for ISR routes", () => { + const before = Date.now(); + const json = buildKVCacheEntryJSON({ kind: "APP_PAGE", html: "", status: 200 }, [], 60); + const after = Date.now(); + const entry = JSON.parse(json); + + // revalidateAt should be approximately now + 60 seconds + expect(entry.revalidateAt).toBeGreaterThanOrEqual(before + 60_000); + expect(entry.revalidateAt).toBeLessThanOrEqual(after + 60_000); + }); + + it("sets revalidateAt to null for static routes", () => { + const json = buildKVCacheEntryJSON({ kind: "APP_PAGE", html: "", status: 200 }, [], false); + const entry = JSON.parse(json); + expect(entry.revalidateAt).toBeNull(); + }); + + it("base64-encodes rscData when present", () => { + const rscBuffer = Buffer.from("RSC payload data"); + const json = buildKVCacheEntryJSON( + { + kind: "APP_PAGE", + html: "", + rscData: rscBuffer, + status: 200, + }, + [], + 60, + ); + const entry = JSON.parse(json); + + expect(typeof entry.value.rscData).toBe("string"); + // Decode and verify round-trip + const decoded = Buffer.from(entry.value.rscData, "base64").toString(); + expect(decoded).toBe("RSC payload data"); + }); + + it("omits rscData from JSON when not provided", () => { + const json = buildKVCacheEntryJSON( + { kind: "APP_PAGE", html: "

test

", status: 200 }, + [], + false, + ); + const entry = JSON.parse(json); + expect(entry.value.rscData).toBeUndefined(); + }); +}); + +// ─── buildRouteEntries ────────────────────────────────────────────────── + +describe("buildRouteEntries", () => { + it("returns 2 pairs when rscBuffer is provided", () => { + const pairs = buildRouteEntries( + "/about", + "

About

", + Buffer.from("rsc-data"), + 60, + "build1", + ); + expect(pairs).toHaveLength(2); + }); + + it("returns 1 pair when rscBuffer is null", () => { + const pairs = buildRouteEntries("/about", "

About

", null, 60, "build1"); + expect(pairs).toHaveLength(1); + }); + + it("produces correct KV keys with ENTRY_PREFIX", () => { + const pairs = buildRouteEntries("/blog/hello", "

content

", Buffer.from("rsc"), 60, "b1"); + const htmlPair = pairs.find((p) => p.key.endsWith(":html")); + const rscPair = pairs.find((p) => p.key.endsWith(":rsc")); + + expect(htmlPair).toBeDefined(); + expect(rscPair).toBeDefined(); + expect(htmlPair!.key).toBe(`${ENTRY_PREFIX}app:b1:/blog/hello:html`); + expect(rscPair!.key).toBe(`${ENTRY_PREFIX}app:b1:/blog/hello:rsc`); + }); + + it("prepends appPrefix when provided", () => { + const pairs = buildRouteEntries("/page", "

hi

", null, 60, "b1", "myapp"); + expect(pairs[0].key).toBe(`myapp:${ENTRY_PREFIX}app:b1:/page:html`); + }); + + it("sets expiration_ttl for ISR routes", () => { + const pairs = buildRouteEntries("/page", "

hi

", null, 60, "b1"); + expect(pairs[0].expiration_ttl).toBe(KV_TTL_SECONDS); + }); + + it("omits expiration_ttl for static routes", () => { + const pairs = buildRouteEntries("/page", "

hi

", null, false, "b1"); + expect(pairs[0].expiration_ttl).toBeUndefined(); + }); + + it("html pair contains APP_PAGE value with html and no rscData", () => { + const pairs = buildRouteEntries("/test", "

Test

", Buffer.from("rsc-payload"), 30, "b1"); + const htmlEntry = JSON.parse(pairs[0].value); + expect(htmlEntry.value.kind).toBe("APP_PAGE"); + expect(htmlEntry.value.html).toBe("

Test

"); + expect(htmlEntry.value.rscData).toBeUndefined(); + }); + + it("rsc pair contains APP_PAGE value with rscData and empty html", () => { + const pairs = buildRouteEntries("/test", "

Test

", Buffer.from("rsc-payload"), 30, "b1"); + const rscEntry = JSON.parse(pairs[1].value); + expect(rscEntry.value.kind).toBe("APP_PAGE"); + expect(rscEntry.value.html).toBe(""); + expect(typeof rscEntry.value.rscData).toBe("string"); + const decoded = Buffer.from(rscEntry.value.rscData, "base64").toString(); + expect(decoded).toBe("rsc-payload"); + }); + + it("includes page tags in both entries", () => { + const pairs = buildRouteEntries("/blog/post", "

post

", Buffer.from("rsc"), 60, "b1"); + const htmlEntry = JSON.parse(pairs[0].value); + const rscEntry = JSON.parse(pairs[1].value); + + const expectedTags = buildPageTags("/blog/post"); + expect(htmlEntry.tags).toEqual(expectedTags); + expect(rscEntry.tags).toEqual(expectedTags); + }); +}); + +// ─── uploadBulkToKV ───────────────────────────────────────────────────── + +describe("uploadBulkToKV", () => { + const baseArgs = { + namespaceId: "ns-123", + accountId: "acc-456", + apiToken: "token-789", + }; + + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("sends a single batch for small payload", async () => { + const pairs: KVBulkPair[] = [ + { key: "cache:app:b1:/:html", value: '{"value":{}}' }, + { key: "cache:app:b1:/:rsc", value: '{"value":{}}' }, + ]; + + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({ success: true }), { status: 200 })); + + await uploadBulkToKV(pairs, baseArgs.namespaceId, baseArgs.accountId, baseArgs.apiToken); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0]; + expect(url).toContain( + `/accounts/${baseArgs.accountId}/storage/kv/namespaces/${baseArgs.namespaceId}/bulk`, + ); + expect(init?.method).toBe("PUT"); + expect(init?.headers).toMatchObject({ + Authorization: `Bearer ${baseArgs.apiToken}`, + "Content-Type": "application/json", + }); + + // Body should be the pairs array + const body = parseFetchBody(init); + expect(body).toHaveLength(2); + expect(body[0].key).toBe("cache:app:b1:/:html"); + }); + + it("splits into multiple batches at 10,000 entries", async () => { + const pairs: KVBulkPair[] = Array.from({ length: 15_000 }, (_, i) => ({ + key: `cache:app:b1:/page-${i}:html`, + value: "{}", + })); + + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({ success: true }), { status: 200 })); + + await uploadBulkToKV(pairs, baseArgs.namespaceId, baseArgs.accountId, baseArgs.apiToken); + + // 15,000 entries → 2 batches (10,000 + 5,000) + expect(fetchSpy).toHaveBeenCalledTimes(2); + + const batch1 = parseFetchBody(fetchSpy.mock.calls[0][1]); + const batch2 = parseFetchBody(fetchSpy.mock.calls[1][1]); + expect(batch1).toHaveLength(10_000); + expect(batch2).toHaveLength(5_000); + }); + + it("throws on API error with descriptive message", async () => { + const pairs: KVBulkPair[] = [{ key: "test", value: "{}" }]; + + vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("Unauthorized", { status: 401 })); + + await expect( + uploadBulkToKV(pairs, baseArgs.namespaceId, baseArgs.accountId, baseArgs.apiToken), + ).rejects.toThrow(/401/); + }); +}); + +// ─── populateKV ───────────────────────────────────────────────────────── + +describe("populateKV", () => { + let tmpDir: string; + let serverDir: string; + let prerenderDir: string; + + beforeEach(() => { + vi.restoreAllMocks(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "populate-kv-test-")); + serverDir = path.join(tmpDir, "dist", "server"); + prerenderDir = path.join(serverDir, "prerendered-routes"); + fs.mkdirSync(prerenderDir, { recursive: true }); + }); + + function writeManifest(manifest: Record) { + fs.writeFileSync(path.join(serverDir, "vinext-prerender.json"), JSON.stringify(manifest)); + } + + function writeHtml(filePath: string, content: string) { + const full = path.join(prerenderDir, filePath); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + + function writeRsc(filePath: string, content: string) { + const full = path.join(prerenderDir, filePath); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + + it("reads manifest and files, produces correct upload payload", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [{ route: "/about", status: "rendered", router: "app", revalidate: 60 }], + }); + writeHtml("about.html", "

About

"); + writeRsc("about.rsc", "rsc-about-data"); + + const uploadedPairs: KVBulkPair[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + const body = parseFetchBody(init); + uploadedPairs.push(...body); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(1); + expect(result.entriesUploaded).toBe(2); // html + rsc + expect(uploadedPairs).toHaveLength(2); + + // Verify keys + expect(uploadedPairs[0].key).toBe("cache:app:b1:/about:html"); + expect(uploadedPairs[1].key).toBe("cache:app:b1:/about:rsc"); + + // Verify HTML entry content + const htmlEntry = JSON.parse(uploadedPairs[0].value); + expect(htmlEntry.value.html).toBe("

About

"); + expect(htmlEntry.value.kind).toBe("APP_PAGE"); + expect(htmlEntry.tags).toEqual(buildPageTags("/about")); + }); + + it("skips routes with status !== rendered", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [ + { route: "/dynamic/[id]", status: "skipped", reason: "dynamic" }, + { route: "/about", status: "rendered", router: "app", revalidate: 60 }, + ], + }); + writeHtml("about.html", "

About

"); + + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ success: true }), { status: 200 }), + ); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(1); + // Only html since no .rsc file + expect(result.entriesUploaded).toBe(1); + }); + + it("skips Pages Router routes", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [ + { route: "/pages-about", status: "rendered", router: "pages", revalidate: 60 }, + { route: "/app-about", status: "rendered", router: "app", revalidate: 60 }, + ], + }); + writeHtml("pages-about.html", "

pages

"); + writeHtml("app-about.html", "

app

"); + + const uploadedPairs: KVBulkPair[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + const body = parseFetchBody(init); + uploadedPairs.push(...body); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + // Only the app route should be processed + expect(result.routesProcessed).toBe(1); + expect(result.entriesUploaded).toBe(1); + expect(uploadedPairs[0].key).toBe("cache:app:b1:/app-about:html"); + }); + + it("skips routes without router field", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [{ route: "/legacy", status: "rendered", revalidate: 60 }], + }); + writeHtml("legacy.html", "

legacy

"); + + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({ success: true }), { status: 200 })); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(0); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("skips routes with missing HTML file", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [{ route: "/ghost", status: "rendered", router: "app", revalidate: 60 }], + }); + // No HTML or RSC files written + + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({ success: true }), { status: 200 })); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(0); + expect(result.entriesUploaded).toBe(0); + // Should not have called the API + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("seeds RSC when .rsc file exists, skips when missing", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [ + { route: "/with-rsc", status: "rendered", router: "app", revalidate: 30 }, + { route: "/no-rsc", status: "rendered", router: "app", revalidate: 30 }, + ], + }); + writeHtml("with-rsc.html", "

with rsc

"); + writeRsc("with-rsc.rsc", "rsc-data"); + writeHtml("no-rsc.html", "

no rsc

"); + // Intentionally no .rsc file for /no-rsc + + const uploadedPairs: KVBulkPair[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + const body = parseFetchBody(init); + uploadedPairs.push(...body); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(2); + expect(result.entriesUploaded).toBe(3); // 2 for with-rsc + 1 for no-rsc + }); + + it("uses path (not route) for dynamic routes", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: false, + routes: [ + { + route: "/blog/[slug]", + status: "rendered", + router: "app", + revalidate: 60, + path: "/blog/hello", + }, + ], + }); + writeHtml("blog/hello.html", "

Hello post

"); + writeRsc("blog/hello.rsc", "rsc-hello"); + + const uploadedPairs: KVBulkPair[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + const body = parseFetchBody(init); + uploadedPairs.push(...body); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + + await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + // Should use /blog/hello (concrete path), not /blog/[slug] (pattern) + expect(uploadedPairs[0].key).toBe("cache:app:b1:/blog/hello:html"); + expect(uploadedPairs[1].key).toBe("cache:app:b1:/blog/hello:rsc"); + }); + + it("returns skipped when manifest is missing", async () => { + // No manifest written + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.skipped).toBeDefined(); + expect(result.routesProcessed).toBe(0); + expect(result.entriesUploaded).toBe(0); + }); + + it("returns skipped when manifest has no buildId", async () => { + writeManifest({ + routes: [{ route: "/about", status: "rendered", router: "app", revalidate: 60 }], + }); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.skipped).toBeDefined(); + }); + + it("handles index route with trailingSlash", async () => { + writeManifest({ + buildId: "b1", + trailingSlash: true, + routes: [{ route: "/about", status: "rendered", router: "app", revalidate: 60 }], + }); + // With trailingSlash, getOutputPath produces about/index.html + writeHtml("about/index.html", "

About

"); + writeRsc("about.rsc", "rsc-data"); + + const uploadedPairs: KVBulkPair[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + const body = parseFetchBody(init); + uploadedPairs.push(...body); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + + const result = await populateKV({ + root: tmpDir, + accountId: "acc", + namespaceId: "ns", + apiToken: "tok", + }); + + expect(result.routesProcessed).toBe(1); + expect(uploadedPairs[0].key).toBe("cache:app:b1:/about:html"); + }); +});