diff --git a/.changeset/adapting-a-starter.md b/.changeset/adapting-a-starter.md new file mode 100644 index 00000000..362e661a --- /dev/null +++ b/.changeset/adapting-a-starter.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": minor +--- + +The skill bundle gains the `adapting-a-starter` recipe: the seven-step transplant procedure for turning an installed starter — a body like vessel-light or the naked skeleton — into your own brand's fingerprint, plus guidance for consuming an unadapted starter honestly. diff --git a/.changeset/body-install.md b/.changeset/body-install.md new file mode 100644 index 00000000..667136cd --- /dev/null +++ b/.changeset/body-install.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": minor +--- + +`ghost init` gains `--body vessel-light`: install a full inhabited fingerprint package — answered signature dials, a materials tree with tokens and embedded fonts, annotated reference compositions, and its own checks. Init template payloads now ship as packed files in dist, so bodies can carry binary materials. diff --git a/.changeset/config.json b/.changeset/config.json index 81dfa838..a43b05c0 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -7,5 +7,5 @@ "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", - "ignore": ["@design-intelligence/vessel", "ghost-docs"] + "ignore": ["@design-intelligence/vessel-react", "ghost-docs"] } diff --git a/.changeset/median-headings-vessel-docs.md b/.changeset/median-headings-vessel-docs.md new file mode 100644 index 00000000..6067b73c --- /dev/null +++ b/.changeset/median-headings-vessel-docs.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": patch +--- + +Reframe the median as Ghost-stamped model truth and align Vessel's median check with heading anchors. diff --git a/.changeset/median-one-home.md b/.changeset/median-one-home.md new file mode 100644 index 00000000..388757b3 --- /dev/null +++ b/.changeset/median-one-home.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": minor +--- + +The median floor gets one authored home: `anti-goal.median` rules become markdown headings with per-rule check references, so `ghost validate` warns on every flag orphaned by pruning. All init templates (`skeleton`, `minimal`, `composition`) now stamp the median node; `ghost checks init` copies the paired check from the packaged payload and skips it when the node is absent. diff --git a/.changeset/review-locator-matching.md b/.changeset/review-locator-matching.md new file mode 100644 index 00000000..c966d0fe --- /dev/null +++ b/.changeset/review-locator-matching.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": patch +--- + +`ghost review` now resolves package-relative `materials/…` locators to their repo-relative form before matching diff paths, so packages living below the repo root offer their value checks correctly. diff --git a/.changeset/skeleton-default-template.md b/.changeset/skeleton-default-template.md new file mode 100644 index 00000000..0323e3b2 --- /dev/null +++ b/.changeset/skeleton-default-template.md @@ -0,0 +1,5 @@ +--- +"@design-intelligence/ghost": minor +--- + +`ghost init` now scaffolds the skeleton starter: value-free grammar nodes, unanswered signature dials, and the `anti-goal.median` floor — replacing the Morrow Ledger demo content. `ghost checks init` scaffolds a live median-tells check alongside the example. `steering` remains as an alias for the new `skeleton` template. diff --git a/.gitignore b/.gitignore index 0c51af86..dbacff6b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,4 @@ packages/ghost-ui/.ghost/cache/* docs/superpowers/plans/ docs/superpowers/specs/ notes/ -demo/ +/demo/ diff --git a/CLAUDE.md b/CLAUDE.md index b4a72c9a..f2bc09ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,7 +89,8 @@ edits and checks. | Package | Published? | Description | | --- | --- | --- | | `packages/ghost` | yes: `@design-intelligence/ghost` | The public package. Ships the `ghost` CLI, node authoring, corpus validation, gather/pull/pulse, review packet assembly, and the unified skill bundle. Shared runtime lives in `packages/ghost/src/ghost-core`. | -| `packages/vessel` | no | A standalone shadcn component registry plus `vessel-mcp` MCP server: the opinionated default reference body. Design-system-agnostic; nothing in Ghost requires it. | +| `packages/vessel-react` | no | A standalone shadcn component registry plus `vessel-mcp` MCP server: the opinionated default reference body. Design-system-agnostic; nothing in Ghost requires it. | +| `packages/vessel-light` | no | Vessel's design language as a portable `.ghost/` fingerprint package for agents writing raw HTML/CSS. No build, no dependencies. | | `apps/docs` | no | Docs site. | ## CLI Commands diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a5140d80..e6f28a5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ the list. ## Dogfooding Surfaces in this repo carry their own fingerprints: `apps/docs/.ghost` and -`packages/vessel/.ghost`. If you touch a surface, `ghost gather` from its +`packages/vessel-react/.ghost`. If you touch a surface, `ghost gather` from its fingerprint before you build, and expect review against it. This is where you experience Ghost as a user — treat friction you hit here as a bug worth reporting. diff --git a/README.md b/README.md index 4a5d326b..83d933a6 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,8 @@ point at things that moved. | Path | Role | Published? | | ---- | ---- | --- | | [`packages/ghost`](./packages/ghost) | The public `ghost` CLI, node authoring, corpus validation, gather/pull, review packet assembly, and the skill bundle. | yes: `@design-intelligence/ghost` | -| [`packages/vessel`](./packages/vessel) | A standalone shadcn component registry plus `vessel-mcp` MCP server. | no | +| [`packages/vessel-react`](./packages/vessel-react) | A standalone shadcn component registry plus `vessel-mcp` MCP server. | no | +| [`packages/vessel-light`](./packages/vessel-light) | Vessel's design language as a portable `.ghost/` fingerprint for agents writing raw HTML/CSS. | no | | [`apps/docs`](./apps/docs) | Docs site. | no | ## Development diff --git a/apps/docs/.ghost/asset.docs-materials.md b/apps/docs/.ghost/asset.docs-materials.md index c2595c21..3994fc9a 100644 --- a/apps/docs/.ghost/asset.docs-materials.md +++ b/apps/docs/.ghost/asset.docs-materials.md @@ -8,8 +8,8 @@ materials: - apps/docs/src/app/**/*.tsx - apps/docs/src/content/docs/*.mdx - apps/docs/src/generated/cli-manifest.json - - packages/vessel/src/styles/main.css - - packages/vessel/src/components/** + - packages/vessel-react/src/styles/main.css + - packages/vessel-react/src/components/** --- Do not invent docs-site vocabulary before inspecting the materials already in @@ -17,8 +17,8 @@ place. Material contract: -- **Tokens and primitives** come from `@design-intelligence/vessel/styles.css`, - backed by `packages/vessel/src/styles/main.css`. Author with semantic roles +- **Tokens and primitives** come from `@design-intelligence/vessel-react/styles.css`, + backed by `packages/vessel-react/src/styles/main.css`. Author with semantic roles (`bg-background`, `text-foreground`, `text-muted-foreground`, `border-border`, `bg-card`) and the named docs typography variables (`--heading-*`, `--label-*`). Avoid raw palette utilities unless they are already part of a diff --git a/apps/docs/.ghost/index.md b/apps/docs/.ghost/index.md index ce3b50ba..cd5150e4 100644 --- a/apps/docs/.ghost/index.md +++ b/apps/docs/.ghost/index.md @@ -9,7 +9,7 @@ materials: This fingerprint governs the Ghost docs site: the landing page, documentation index, tool landings, MDX docs pages, and the small components that carry them. It does not govern Vessel as a reference component registry; Vessel has its own -fingerprint under `packages/vessel/.ghost/`. When the docs consume Vessel +fingerprint under `packages/vessel-react/.ghost/`. When the docs consume Vessel materials, this fingerprint decides the docs' product stance and Vessel decides the component-system contract. diff --git a/apps/docs/.ghost/principle.visual-composition.md b/apps/docs/.ghost/principle.visual-composition.md index 39b52795..3fb9c6a7 100644 --- a/apps/docs/.ghost/principle.visual-composition.md +++ b/apps/docs/.ghost/principle.visual-composition.md @@ -5,7 +5,7 @@ materials: - apps/docs/src/app/docs/page.tsx - apps/docs/src/app/tools/**/*.tsx - apps/docs/src/components/docs/** - - packages/vessel/src/styles/main.css + - packages/vessel-react/src/styles/main.css --- The Ghost docs site should render like a stark technical artifact with one diff --git a/apps/docs/README.md b/apps/docs/README.md index c3a3d0c4..062e28e6 100644 --- a/apps/docs/README.md +++ b/apps/docs/README.md @@ -2,7 +2,7 @@ **Documentation site for the Ghost project.** -`ghost-docs` is the deployed docs for everything in this monorepo: the `ghost` CLI, the fingerprint package format, the generation loop, and the live `vessel` component catalogue. A Vite + MDX app that consumes [`vessel`](../../packages/vessel) as a workspace dependency. +`ghost-docs` is the deployed docs for everything in this monorepo: the `ghost` CLI, the fingerprint package format, the generation loop, and the live `vessel` component catalogue. A Vite + MDX app that consumes [`vessel-react`](../../packages/vessel-react) as a workspace dependency. ## Run diff --git a/apps/docs/package.json b/apps/docs/package.json index ffbca9dd..d8e61593 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@design-intelligence/vessel": "workspace:*", + "@design-intelligence/vessel-react": "workspace:*", "@hookform/resolvers": "^5.2.2", "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-alert-dialog": "^1.1.15", diff --git a/apps/docs/src/App.tsx b/apps/docs/src/App.tsx index 23bc35da..e66db5df 100644 --- a/apps/docs/src/App.tsx +++ b/apps/docs/src/App.tsx @@ -1,4 +1,4 @@ -import { ThemeProvider } from "@design-intelligence/vessel"; +import { ThemeProvider } from "@design-intelligence/vessel-react"; import { useEffect } from "react"; import { Navigate, Route, Routes, useLocation } from "react-router"; import DocsIndex from "@/app/docs/page"; diff --git a/apps/docs/src/app/docs/page.tsx b/apps/docs/src/app/docs/page.tsx index 78624df5..2ffae40e 100644 --- a/apps/docs/src/app/docs/page.tsx +++ b/apps/docs/src/app/docs/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useStaggerReveal } from "@design-intelligence/vessel"; +import { useStaggerReveal } from "@design-intelligence/vessel-react"; import { BookOpen, FileText, Rocket, ShieldCheck, Zap } from "lucide-react"; import type { ReactNode } from "react"; import { Link } from "react-router"; diff --git a/apps/docs/src/app/page.tsx b/apps/docs/src/app/page.tsx index 67d599df..7b80cec0 100644 --- a/apps/docs/src/app/page.tsx +++ b/apps/docs/src/app/page.tsx @@ -1,4 +1,4 @@ -import { useStaggerReveal } from "@design-intelligence/vessel"; +import { useStaggerReveal } from "@design-intelligence/vessel-react"; import { Hero } from "@/components/docs/hero"; import { SectionWrapper } from "@/components/docs/wrappers"; diff --git a/apps/docs/src/app/tools/drift/page.tsx b/apps/docs/src/app/tools/drift/page.tsx index 98bfda0e..9b977ef9 100644 --- a/apps/docs/src/app/tools/drift/page.tsx +++ b/apps/docs/src/app/tools/drift/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useStaggerReveal } from "@design-intelligence/vessel"; +import { useStaggerReveal } from "@design-intelligence/vessel-react"; import { BookOpen, Orbit, Rocket } from "lucide-react"; import type { ReactNode } from "react"; import { Link } from "react-router"; diff --git a/apps/docs/src/app/tools/page.tsx b/apps/docs/src/app/tools/page.tsx index f6844569..c5881296 100644 --- a/apps/docs/src/app/tools/page.tsx +++ b/apps/docs/src/app/tools/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useStaggerReveal } from "@design-intelligence/vessel"; +import { useStaggerReveal } from "@design-intelligence/vessel-react"; import { FileText, Orbit } from "lucide-react"; import type { ReactNode } from "react"; import { Link } from "react-router"; diff --git a/apps/docs/src/app/tools/scan/page.tsx b/apps/docs/src/app/tools/scan/page.tsx index eaf2debf..43829114 100644 --- a/apps/docs/src/app/tools/scan/page.tsx +++ b/apps/docs/src/app/tools/scan/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useStaggerReveal } from "@design-intelligence/vessel"; +import { useStaggerReveal } from "@design-intelligence/vessel-react"; import { BookOpen, FileText, Rocket } from "lucide-react"; import type { ReactNode } from "react"; import { Link } from "react-router"; diff --git a/apps/docs/src/components/docs/callout.tsx b/apps/docs/src/components/docs/callout.tsx index 205fbbf7..5f2443bb 100644 --- a/apps/docs/src/components/docs/callout.tsx +++ b/apps/docs/src/components/docs/callout.tsx @@ -1,4 +1,4 @@ -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import type { ReactNode } from "react"; type Variant = "info" | "warning" | "wip"; diff --git a/apps/docs/src/components/docs/doc-prose.tsx b/apps/docs/src/components/docs/doc-prose.tsx index 886f9605..7ffbea7e 100644 --- a/apps/docs/src/components/docs/doc-prose.tsx +++ b/apps/docs/src/components/docs/doc-prose.tsx @@ -1,4 +1,4 @@ -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import type { ComponentProps } from "react"; export function DocProse({ diff --git a/apps/docs/src/components/docs/dock.tsx b/apps/docs/src/components/docs/dock.tsx index d07e7964..64ca8318 100644 --- a/apps/docs/src/components/docs/dock.tsx +++ b/apps/docs/src/components/docs/dock.tsx @@ -12,7 +12,7 @@ import { TooltipContent, TooltipTrigger, useTheme, -} from "@design-intelligence/vessel"; +} from "@design-intelligence/vessel-react"; import type { LucideIcon } from "lucide-react"; import { BookOpen, diff --git a/apps/docs/src/components/docs/docs-page-layout.tsx b/apps/docs/src/components/docs/docs-page-layout.tsx index ae387ebb..e92a7d30 100644 --- a/apps/docs/src/components/docs/docs-page-layout.tsx +++ b/apps/docs/src/components/docs/docs-page-layout.tsx @@ -1,6 +1,6 @@ "use client"; -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import type { ReactNode } from "react"; /** diff --git a/apps/docs/src/components/docs/external-link.tsx b/apps/docs/src/components/docs/external-link.tsx index ae739801..f6c891bf 100644 --- a/apps/docs/src/components/docs/external-link.tsx +++ b/apps/docs/src/components/docs/external-link.tsx @@ -1,4 +1,4 @@ -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import { ArrowUpRight } from "lucide-react"; import { ComponentProps } from "react"; diff --git a/apps/docs/src/components/docs/page-header.tsx b/apps/docs/src/components/docs/page-header.tsx index 826f5e32..c122d6f2 100644 --- a/apps/docs/src/components/docs/page-header.tsx +++ b/apps/docs/src/components/docs/page-header.tsx @@ -1,4 +1,4 @@ -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import { ComponentProps } from "react"; function PageHeader({ diff --git a/apps/docs/src/components/docs/tooltip-wrapper.tsx b/apps/docs/src/components/docs/tooltip-wrapper.tsx index 71a32704..8abd06e1 100644 --- a/apps/docs/src/components/docs/tooltip-wrapper.tsx +++ b/apps/docs/src/components/docs/tooltip-wrapper.tsx @@ -5,7 +5,7 @@ import { Tooltip, TooltipContent, TooltipTrigger, -} from "@design-intelligence/vessel"; +} from "@design-intelligence/vessel-react"; import { ComponentProps } from "react"; import { useShowTooltips } from "@/store/preferences-store"; diff --git a/apps/docs/src/components/docs/wrappers.tsx b/apps/docs/src/components/docs/wrappers.tsx index 4a08dab0..18fad502 100644 --- a/apps/docs/src/components/docs/wrappers.tsx +++ b/apps/docs/src/components/docs/wrappers.tsx @@ -1,4 +1,4 @@ -import { cn } from "@design-intelligence/vessel"; +import { cn } from "@design-intelligence/vessel-react"; import { ComponentProps } from "react"; interface SectionWrapperProps extends ComponentProps<"div"> { diff --git a/apps/docs/src/content/docs/cli-reference.mdx b/apps/docs/src/content/docs/cli-reference.mdx index 8151abe1..26d17987 100644 --- a/apps/docs/src/content/docs/cli-reference.mdx +++ b/apps/docs/src/content/docs/cli-reference.mdx @@ -51,12 +51,29 @@ a finding (exit `1`). ### Initialize: `init` -Scaffold a `.ghost/` package with the steering starter: a manifest, a starter `glossary.md`, a core `index.md` node, and demo nodes for stance, composition, anti-goals, patterns, exemplars, materials, and decisions. The demo content is there for inspiration and guidance. Replace it with real product truth before using it to steer generation. Use `--template minimal` when you only want the small manifest/glossary/index starter. Add truths by adding files (`principle.density.md`). Use +Scaffold a `.ghost/` package. Every template and body includes `anti-goal.median`; the skeleton starter also includes a manifest, a `glossary.md` declaring the grammar/signature/register/anti-goal kinds, a core `index.md`, six grammar nodes of value-free decision logic, and four signature nodes as unanswered dials awaiting your brand's values. The median node is Ghost's measured knowledge of unsteered model behavior — model truth, not brand truth — stamped at init and owned (pruned, adapted) by you thereafter. Nothing in the skeleton is fiction: grammar and the median floor are safe to consume verbatim; the dials tell the agent to ask or flag, never freehand. Use `--template minimal` when you only want the small manifest/glossary/index starter. Add truths by adding files (`principle.density.md`). Use `--package ` for an exact directory, or set `GHOST_PACKAGE_DIR` when a host wrapper stores Ghost files outside the default `.ghost`. Core `init` scaffolds the fingerprint only; pass `--with checks` to also add the checks directory in one move. +Templates are shapes of emptiness — structures awaiting your truths. A +**body** is the same anatomy inhabited: answered signature dials, a +`materials/` tree (tokens, fonts, worked reference compositions), and its own +`checks/`. Install one with `--body`: + +```bash +ghost init --body vessel-light +``` + +`vessel-light` installs Vessel's design language at full strength — the +fork-axis corpus, `materials/tokens.css` owning every literal value, embedded +fonts, six annotated refs, and five checks, included automatically (no +`--with checks` needed; combining them is an error). The manifest id stays +`vessel-light` until you adapt the starter: renaming it is deliberately the +first step of making the brand yours. `--body` and `--template` are mutually +exclusive. + ```bash diff --git a/apps/docs/src/content/docs/getting-started.mdx b/apps/docs/src/content/docs/getting-started.mdx index 023f0217..3dd3cebe 100644 --- a/apps/docs/src/content/docs/getting-started.mdx +++ b/apps/docs/src/content/docs/getting-started.mdx @@ -30,7 +30,7 @@ assembles advisory review packets after, but never grades output itself. You work through a short loop: ```bash -ghost init # scaffold .ghost/ with the steering starter +ghost init # scaffold .ghost/ with the skeleton starter ghost validate # artifact shape + node validity + kind-prefix check ghost gather # emit the menu for this task; your agent selects the truths it needs ghost pull # read the selected truths' full bodies @@ -80,13 +80,25 @@ by decision. One high-confidence truth beats an empty catalog. The CLI handles the deterministic package work. Your agent handles the authoring: interviewing you, working with the material you show it, drafting node prose, -and asking you to curate the claims. `ghost init` scaffolds the steering starter: `manifest.yml`, a starter `glossary.md`, a core `index.md` node, and demo nodes for stance, composition, anti-goals, patterns, exemplars, materials, and decisions. The demo content is there for inspiration and guidance. Replace its claims, paths, examples, and decisions with your real product truth before using it to steer generation. Use `ghost init --template minimal` when you only want the small manifest/glossary/index starter. +and asking you to curate the claims. `ghost init` stamps `anti-goal.median` into every template and body; the median node is Ghost's measured knowledge of unsteered model behavior — model truth, not brand truth — owned (pruned, adapted) by you after init. The skeleton starter also includes `manifest.yml`, a `glossary.md` declaring the grammar/signature/register/anti-goal kinds, a core `index.md`, six grammar nodes of value-free decision logic, and four signature nodes as unanswered dials. Nothing in the skeleton is fiction: grammar and the median floor hold for any brand, and each dial tells the agent to ask you or flag the gap rather than freehand a value. Answering the dials is how the skeleton becomes your brand. Use `ghost init --template minimal` when you only want the small manifest/glossary/index starter. ```bash ghost init ghost validate ``` +Prefer starting from a working brand instead of an empty anatomy? Install a +**body** — the same skeleton with a real brand's values plugged in: answered +dials, a `materials/` tree with tokens and fonts, worked reference +compositions, and its own checks: + +```bash +ghost init --body vessel-light +``` + +The body's manifest keeps its own id (`vessel-light`) until you adapt it — +renaming the id is deliberately the first step of making the brand yours. + Each node's prose records a durable brand truth. While drafting, ask three questions of every truth: **why** (the stance — what this brand is, who it serves, what must remain true), **with what** (the materials — building blocks, diff --git a/apps/docs/src/generated/cli-manifest.json b/apps/docs/src/generated/cli-manifest.json index 02c31fa7..99351c47 100644 --- a/apps/docs/src/generated/cli-manifest.json +++ b/apps/docs/src/generated/cli-manifest.json @@ -1,5 +1,5 @@ { - "generatedAt": "2026-07-07T16:48:56.039Z", + "generatedAt": "2026-07-09T11:06:22.019Z", "tools": [ { "tool": "ghost", @@ -53,7 +53,15 @@ { "rawName": "--template ", "name": "template", - "description": "Init template to scaffold (default: steering)", + "description": "Init template to scaffold (default: skeleton)", + "default": null, + "takesValue": true, + "negated": false + }, + { + "rawName": "--body ", + "name": "body", + "description": "Init body to install: a full inhabited package (e.g. vessel-light)", "default": null, "takesValue": true, "negated": false diff --git a/apps/docs/src/main.tsx b/apps/docs/src/main.tsx index 8c6c9d87..061e9451 100644 --- a/apps/docs/src/main.tsx +++ b/apps/docs/src/main.tsx @@ -1,7 +1,7 @@ import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { BrowserRouter } from "react-router"; -import "@design-intelligence/vessel/styles.css"; +import "@design-intelligence/vessel-react/styles.css"; import { App } from "./App"; import "./styles/dev-fonts.css"; diff --git a/biome.json b/biome.json index 43ba5ec2..43067a1d 100644 --- a/biome.json +++ b/biome.json @@ -6,7 +6,7 @@ "useIgnoreFile": true }, "files": { - "includes": ["**", "!**/test/fixtures"] + "includes": ["**", "!**/test/fixtures", "!packages/vessel-light/demo/out"] }, "formatter": { "enabled": true, @@ -35,7 +35,7 @@ }, "overrides": [ { - "includes": ["packages/vessel/**", "apps/docs/**"], + "includes": ["packages/vessel-react/**", "apps/docs/**"], "linter": { "enabled": false } diff --git a/install/manifest.json b/install/manifest.json index 2595f730..951098a7 100644 --- a/install/manifest.json +++ b/install/manifest.json @@ -8,6 +8,7 @@ }, "files": [ "SKILL.md", + "references/adapting-a-starter.md", "references/authoring-scenarios.md", "references/blocks.md", "references/brief.md", diff --git a/package.json b/package.json index 72af3b23..d480332b 100644 --- a/package.json +++ b/package.json @@ -8,15 +8,15 @@ "node": "^20.19.0 || >=22.12.0" }, "scripts": { - "build": "tsc --build --force", + "build": "tsc --build --force && pnpm --filter @design-intelligence/ghost build", "clean": "tsc --build --clean", "test": "vitest run", "test:watch": "vitest", "typecheck": "tsc --build", - "build:ui": "pnpm --filter @design-intelligence/vessel build", + "build:ui": "pnpm --filter @design-intelligence/vessel-react build", "check": "biome check . && pnpm typecheck && pnpm check:package-bin && pnpm check:packed-package && pnpm check:file-sizes && pnpm check:terminology && pnpm check:docs && pnpm check:install-bundle && pnpm check:release-tarball && pnpm check:release-workflows && pnpm check:cli-manifest", "check:file-sizes": "node scripts/check-file-sizes.mjs", - "check:packed-package": "node scripts/check-packed-package.mjs", + "check:packed-package": "pnpm --filter @design-intelligence/ghost build && node scripts/check-packed-package.mjs", "check:package-bin": "chmod +x packages/ghost/dist/bin.js && node scripts/link-package-bin.mjs && node scripts/check-package-bin.mjs", "check:terminology": "node scripts/check-terminology.mjs", "check:docs": "node scripts/check-docs-frontmatter.mjs", diff --git a/packages/ghost/package.json b/packages/ghost/package.json index 18a4fe60..fb7a8ca5 100644 --- a/packages/ghost/package.json +++ b/packages/ghost/package.json @@ -69,7 +69,7 @@ "provenance": true }, "scripts": { - "build": "rm -rf dist tsconfig.tsbuildinfo && tsc --build --force && chmod +x dist/bin.js && node ../../scripts/link-package-bin.mjs && cp -r src/skill-bundle dist/skill-bundle", + "build": "rm -rf dist tsconfig.tsbuildinfo && tsc --build --force && chmod +x dist/bin.js && node ../../scripts/link-package-bin.mjs && cp -r src/skill-bundle dist/skill-bundle && node ../../scripts/sync-init-payloads.mjs", "prepublishOnly": "pnpm build" }, "dependencies": { diff --git a/packages/ghost/src/commands/checks-command.ts b/packages/ghost/src/commands/checks-command.ts index e1dd4b32..2fa22d7a 100644 --- a/packages/ghost/src/commands/checks-command.ts +++ b/packages/ghost/src/commands/checks-command.ts @@ -35,7 +35,11 @@ export function registerChecksCommand(cli: CAC): void { if (opts.format === "json") { process.stdout.write( `${JSON.stringify( - { dir: result.dir, written: result.written }, + { + dir: result.dir, + written: result.written, + skipped: result.skipped, + }, null, 2, )}\n`, @@ -45,6 +49,9 @@ export function registerChecksCommand(cli: CAC): void { for (const file of result.written) { process.stdout.write(` ${file}\n`); } + for (const file of result.skipped) { + process.stdout.write(` skipped ${file}\n`); + } } process.exit(0); } catch (err) { diff --git a/packages/ghost/src/commands/init-command.ts b/packages/ghost/src/commands/init-command.ts index a9d41633..02946d3c 100644 --- a/packages/ghost/src/commands/init-command.ts +++ b/packages/ghost/src/commands/init-command.ts @@ -3,6 +3,7 @@ import { UsageError } from "#ghost-core"; import { initFingerprintPackage } from "../fingerprint.js"; import { addChecksDir } from "../scan/check-scaffold.js"; import { resolveGhostDirDefault } from "../scan/index.js"; +import { getInitBody } from "../scan/templates.js"; import { failFromError } from "./errors.js"; export function registerInitCommand(cli: CAC): void { @@ -14,7 +15,11 @@ export function registerInitCommand(cli: CAC): void { ) .option( "--template ", - "Init template to scaffold (default: steering)", + "Init template to scaffold (default: skeleton)", + ) + .option( + "--body ", + "Init body to install: a full inhabited package (e.g. vessel-light)", ) .option( "--with ", @@ -35,6 +40,22 @@ export function registerInitCommand(cli: CAC): void { typeof opts.package === "string" ? opts.package : undefined; const ghostDir = exactPackage === undefined ? ghostDirFromEnv() : undefined; + const withIds = parseWithCapabilities(opts.with); + for (const id of withIds) { + if (id !== "checks") { + throw new UsageError( + `Unknown --with capability '${id}'. Available: checks.`, + ); + } + } + const body = + typeof opts.body === "string" ? getInitBody(opts.body) : undefined; + if (body?.includesChecks && withIds.includes("checks")) { + throw new UsageError( + `--with checks is redundant with --body ${body.name} — this body already includes its own checks/.`, + ); + } + const result = await initFingerprintPackage( exactPackage ?? ghostDir, process.cwd(), @@ -42,18 +63,10 @@ export function registerInitCommand(cli: CAC): void { ...(typeof opts.template === "string" ? { template: opts.template } : {}), + ...(typeof opts.body === "string" ? { body: opts.body } : {}), force: Boolean(opts.force), }, ); - - const withIds = parseWithCapabilities(opts.with); - for (const id of withIds) { - if (id !== "checks") { - throw new UsageError( - `Unknown --with capability '${id}'. Available: checks.`, - ); - } - } const addedChecks = withIds.includes("checks") ? await addChecksDir(result.paths.packageDir) : undefined; @@ -65,7 +78,12 @@ export function registerInitCommand(cli: CAC): void { dir: result.paths.dir, written: result.written, ...(addedChecks !== undefined - ? { checks: { written: addedChecks.written } } + ? { + checks: { + written: addedChecks.written, + skipped: addedChecks.skipped, + }, + } : {}), }, null, @@ -84,6 +102,9 @@ export function registerInitCommand(cli: CAC): void { for (const file of addedChecks.written) { process.stdout.write(` checks/${file}\n`); } + for (const file of addedChecks.skipped) { + process.stdout.write(` skipped checks/${file}\n`); + } } } process.exit(0); diff --git a/packages/ghost/src/commands/review-command.ts b/packages/ghost/src/commands/review-command.ts index eb314d3b..730e027c 100644 --- a/packages/ghost/src/commands/review-command.ts +++ b/packages/ghost/src/commands/review-command.ts @@ -53,6 +53,7 @@ export function registerReviewCommand(cli: CAC): void { diff: opts.diff, }); const packet = await buildReviewPacket(fingerprint, diffText, { + packageDir: paths.packageDir, runProbes: opts.probes !== false, cwd: process.cwd(), }); diff --git a/packages/ghost/src/init-payloads/median/anti-goal.median.md b/packages/ghost/src/init-payloads/median/anti-goal.median.md new file mode 100644 index 00000000..a0a5cefb --- /dev/null +++ b/packages/ghost/src/init-payloads/median/anti-goal.median.md @@ -0,0 +1,86 @@ +--- +description: "The model's median defaults this fingerprint refuses — gather for any greenfield visual surface or first-draft copy. Each rule is reject→replace; delete lines your brand legitimately violates." +--- + +This is the model's median, not your brand. Each rule is reject→replace. +Delete every line your brand legitimately violates — `ghost validate` will +warn on any check reference the deletion orphans — delete the paired flag too. + +These are not aesthetic opinions. Where a count is given, it is the measured +convergence of 300 unsteered generations across three frontier models (the +antimedian experiment): the defaults a model reaches for when nobody hands it +a brand. An output showing several of these tells reads as generated, +whatever else it does right. A deterministic floor rides alongside these +rules in the paired check — mechanical tells like gradient-filled text, +emoji as interface icons, em-dash pileups, and aphoristic cadence — +verified at review, never steered in prose. + +### Hover-lift +Reject hover-lift (`translateY` + growing shadow) as the default interaction +→ confirm with color and background change at the fingerprint's fast +duration. (measured: 341) + +### Indigo accent +Reject the indigo/blue/purple default accent (`#4f46e5`, `#2563eb`, +`#8b5cf6` family; purple/violet hue 260–310) → the fingerprint's declared +palette; absent one, monochrome plus a single functional accent. + +### Dark theme +Reject unprompted dark theme → the fingerprint's declared surface. +Dark-to-look-cool and light-to-be-safe are the same retreat from a decision. +(measured: 271) + +### Gradients +Reject gradient page and section backgrounds and gradient-filled CTAs → flat +surfaces in semantic roles. + +### Side-stripe +Reject side-stripe borders (a thick colored border on one side of a rounded +card) — the single most recognizable tell of AI-generated UI → a full +hairline border, a 4–8% surface tint, or a leading glyph. + +### Cream surface +Reject the cream/sand/beige default surface (warm off-white; token names +like `--cream`, `--sand`, `--parchment` are the tell) → a true off-white, +the brand's own hue, or a committed color; warmth via accent and type, not +the ground. + +### Glassmorphism +Reject glassmorphism (decorative backdrop blur) → flat surfaces with real +borders and the fingerprint's elevation tiers. + +### Chat bubbles +Reject chat bubbles with initials-circle avatars for assistant turns → +assistant text plain on the page surface; user turns marked compactly. + +### Stock copy +Reject stock template copy ("Simple, transparent pricing") → headings that +state what this product specifically does. + +### Celebration +Reject celebration copy and UI (exclamation success, confetti language) → +quiet factual confirmation. + +### Nested cards +Reject nested cards and everything-in-cards → one surface level per region; +interior hierarchy from spacing and type; spacing and alignment group +without card overhead. + +### Hero metric +Reject the hero-metric template (big number + small label + supporting stats +as default proof) → evidence specific to the product, or nothing. A +prominent metric showing real user data is fine. + +### Every button primary +Reject every button a primary button → one primary per view; the rest step +down the fingerprint's emphasis ladder. + +### Eyebrow kicker +Reject an eyebrow kicker on every section → at most one, where the register +sanctions it; one named kicker is voice, every-section is AI grammar. + +### Decorative motion +Reject decorative looping animation (pulse/float/shimmer) and uniform +fade-and-rise on every scrolled section → motion as evidence of state +change; loops only for genuine loading; decoration never compensates for +weak structure — fix spacing, copy, and hierarchy first. diff --git a/packages/ghost/src/init-payloads/median/median-tells.md b/packages/ghost/src/init-payloads/median/median-tells.md new file mode 100644 index 00000000..fd8e8acd --- /dev/null +++ b/packages/ghost/src/init-payloads/median/median-tells.md @@ -0,0 +1,117 @@ +--- +name: Median tells +description: Flags the measured defaults of unsteered generation, the deterministic floor, and current model-signature tells — hover-lift, default accents, unprompted dark theme, gradient text, contrast, frequency tells, and per-model signatures. +severity: high +references: + - anti-goal.median > Hover-lift + - anti-goal.median > Indigo accent + - anti-goal.median > Dark theme + - anti-goal.median > Gradients + - anti-goal.median > Glassmorphism + - anti-goal.median > Side-stripe + - anti-goal.median > Cream surface + - anti-goal.median > Chat bubbles + - anti-goal.median > Stock copy + - anti-goal.median > Celebration + - anti-goal.median > Hero metric + - anti-goal.median > Eyebrow kicker +--- + +These flags target the measured convergence patterns of unsteered model +generation, the deterministic floor the median node licenses, and tells +specific to individual models. Each is mechanically detectable in a diff. +Pruning a rule from `anti-goal.median` orphans its paired reference here — +`ghost validate` warns; delete the flag and its reference together. + +Flag `transform` with `translateY` inside a `:hover` rule on cards, +buttons, or list items, especially paired with a shadow increase. Hover +confirmation in this fingerprint is color and background change, not lift. +(`anti-goal.median > Hover-lift`) + +Flag accent values in the indigo/blue/purple default family (`#4f46e5`, +`#6366f1`, `#2563eb`, `#3b82f6`, `#8b5cf6`, and close neighbors) +unless the diff shows the user asked for them. They are model defaults, not +palette members. (`anti-goal.median > Indigo accent`) + +Flag whole-page dark backgrounds when the ask did not request dark mode. +Dark surfaces are a declared brand choice or an explicit theme, never an +unprompted default. (`anti-goal.median > Dark theme`) + +Flag `linear-gradient` or `radial-gradient` as page or section +backgrounds, and gradient-filled buttons. (`anti-goal.median > Gradients`) + +Flag `backdrop-filter: blur` used for glassmorphism cards. +(`anti-goal.median > Glassmorphism`) + +Flag `background-clip: text` (with or without the `-webkit-` prefix) +paired with a gradient. Emphasis comes from weight or size in a single +solid color. + +Flag a thick colored border on one side of an element (`border-left` or a +`border-l-*` utility at 2px or more in a non-neutral color) while the +other sides stay thin. (`anti-goal.median > Side-stripe`) + +Flag warm off-white page backgrounds in the cream/sand/beige band, and token +names like `--cream`, `--sand`, `--parchment`, `--linen` introduced +by the diff. (`anti-goal.median > Cream surface`) + +Flag assistant messages rendered as bubbles with initials-circle avatars. +(`anti-goal.median > Chat bubbles`) + +Flag emoji used as icons or imagery in interface chrome. Text labels carry +meaning. + +Flag stock template copy in headings: "Simple, transparent pricing", +"Welcome back", and interchangeable-with-a-competitor phrasing. Recommend +copy that states what this product specifically does. +(`anti-goal.median > Stock copy`) + +Flag exclamation-marked success copy, confetti language, and celebratory UI +("You did it!", "Awesome!"). Confirmation is quiet and factual. +(`anti-goal.median > Celebration`) + +Flag the hero-metric template — a big number, small label, and supporting +stats as default proof — unless the metric shows real user data. Recommend +evidence specific to the product, or nothing. +(`anti-goal.median > Hero metric`) + +Deterministic floor — licensed by the median node, verified here, never +steered in prose: + +Flag text/background pairs below WCAG AA contrast: 4.5:1 for body text, +3:1 for large text (24px and up, or 18.7px bold and up). Check the worst +stop when the background is a gradient. + +Flag animation or transition without a `prefers-reduced-motion: reduce` +alternative — a crossfade or instant state change. + +Flag ad-hoc z-index values (`999`, `9999`, or any value outside a +declared scale). Layering wants a semantic scale, not an arms race. + +Flag `cubic-bezier` easings whose control points overshoot the 0–1 range, +and keyframe names matching bounce, elastic, wobble, or jiggle. + +Frequency tells — the crime is repetition, not the move (advisory): + +Flag three or more uppercase, tracked eyebrow kickers above section headings +in one page. One named kicker is voice; a kicker on every section is model +grammar. (`anti-goal.median > Eyebrow kicker`) + +Flag five or more em-dashes in body copy in one view. + +Flag three or more instances of the aphoristic rebuttal cadence ("Not X. Y." +/ "Sentence. No qualifiers.") in one page's copy. + +Model-signature tells — skip any block whose model did not produce the diff +(advisory): + +Codex: flag a 1px border paired with a box-shadow of 16px blur or more on +the same element — pick a solid border or a tight shadow, not both. Flag +border-radius of 32px or more on cards, sections, or inputs unless an +answered shape dial (`signature.shape`) sanctions large radii — then it is +fidelity, not a tell. Flag 1px linear-gradient grid or repeating-stripe +backgrounds used as decoration. + +Gemini: flag `transform` (scale, rotate, translate) on `img` inside a +`:hover` rule, including group-hover utilities targeting a child image. +Animate the card's background, border, or shadow instead. diff --git a/packages/ghost/src/init-payloads/skeleton/glossary.md b/packages/ghost/src/init-payloads/skeleton/glossary.md new file mode 100644 index 00000000..053ffb08 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/glossary.md @@ -0,0 +1,44 @@ +--- +kinds: + - name: grammar + - name: signature + - name: register + - name: anti-goal + posture: guard +--- + +# grammar + +The brand's decision logic: closed sets, role vocabularies, and assembly +rules, stated in token roles and never in literal values. Grammar survives an +adaptation unchanged — swap every value and these nodes still hold. Gather +grammar before inventing structure. + +# signature + +The dials: the choices that make a brand this brand. In this starter every +signature node is unanswered — it states the fixed relationship worth +keeping and asks the question only a human can answer. When the human +answers, restate the node as the brand's current answer. Gather signature +before setting any value a dial governs. + +# register + +A condition-scoped contract: a situation — data-dense consoles, editorial +pages, transactional email — where parts of the default rules invert. Each +register names its condition first. Truths from the wrong register are +poison, not context; gather a register only when its condition matches the +task. This starter declares the kind and ships no register nodes: registers +invert answered defaults, and the dials are not answered yet. + +# anti-goal + +What this package refuses, each reject paired with its replacement. +`anti-goal.median` is the model's floor, not the brand's taste. Gather +anti-goals before styling anything greenfield. + +--- + +The ghost skill bundle teaches authoring — its `adapting-a-starter` recipe is +the procedure for growing this skeleton into a real fingerprint: answer the +dials, prune the median floor, edit your values, then generate refs and tells. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.color-roles.md b/packages/ghost/src/init-payloads/skeleton/grammar.color-roles.md new file mode 100644 index 00000000..eb4f679d --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.color-roles.md @@ -0,0 +1,26 @@ +--- +description: "Semantic color roles and the rules that govern them — gather before choosing any color; roles not raw values, status is meaning, expression is register-gated and never on controls." +--- + +The token roles are the source of truth. An agent may combine roles, but it +may not author around them. Raw color values are implementation detail, never +product language. + +Author with semantic roles: `background`, `foreground`, `card`, `popover`, +`primary`, `secondary`, `muted`, `accent`, `border`, `input`, `ring`, and the +status roles. If a container needs a color, it first needs a role. + +The status roles — destructive, success, warning, info — exist only when +meaning demands them. Destructive means destructive or error. Success means +success. Warning means warning. Info means information. None of them are +brand accents, and they never moonlight as atmosphere, in any register. + +One view should not perform a color palette. If a status color is present, +let the rest of the view stay on the base roles. Richness beyond this is +register-gated: a closed expression set (`--expression-*`) exists; its size +and members are a dial — see the palette signature — and each register caps +how loud they may be. + +The constant that holds across every register: expression never touches what +you click. Buttons, inputs, and links stay on the base roles everywhere. A +colored control is a different design system. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.conversation.md b/packages/ghost/src/init-payloads/skeleton/grammar.conversation.md new file mode 100644 index 00000000..fc2e1b63 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.conversation.md @@ -0,0 +1,23 @@ +--- +description: "Conversation grammar — plain assistant text, compact user surfaces, collapsed tool calls, one structured prompt input; gather for any AI thread, agent console, review assistant, or prompt composer." +--- + +Conversation UI is not chat cosplay. The assistant speaks on the page surface +as plain text: no bubble, no border, no fill. + +User turns are compact muted surfaces aligned right. They mark authorship +without turning the thread into alternating balloons. + +Assistant hierarchy comes from prose, spacing, and type. Wrapping assistant +messages in cards makes the system look defensive and wastes density. + +Tool calls are operational evidence. Collapse them to a labeled one-line +summary with status. Expand only when the user asks for detail, then show +mono content inside the disclosed area. + +The prompt input is one bordered surface. The textarea region stays empty of +controls so writing remains the focus. Attachments, model choices, secondary +tools, and send live in a single row below it. + +There is one primary send action. Stop and send are mutually exclusive states +of the same action area, not two competing primary buttons. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.hierarchy.md b/packages/ghost/src/init-payloads/skeleton/grammar.hierarchy.md new file mode 100644 index 00000000..97f2c62b --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.hierarchy.md @@ -0,0 +1,39 @@ +--- +description: "The closed hierarchy vocabulary — six text variants, seven tones, a five-rung control emphasis ladder, one primary per view — gather for any view that contains text or actions." +--- + +Hierarchy is a closed vocabulary, not a size slider. Every piece of text and +every control picks from a named set; anything outside the set is a broken +primitive, not a variant. + +The text variants are exactly six: display, headline, title, body, label, +mono. Variant names are jobs, not decoration. Display leads a rare editorial +moment. Headline names a section. Title anchors a card, dialog, or compact +region. Body carries reading. Label names structure — category tags, field +labels, bylines, metadata, compact status. Mono carries code and machine +detail. Do not fake hierarchy with arbitrary font sizes; choose the tier that +matches the job, then use tone, weight, and spacing for the rest. + +The tones are exactly seven: default, muted, inverse, success, warning, info, +destructive. Tone is part of the message. Default is the main reading plane. +Muted carries secondary information. Inverse is for dark or primary surfaces. +The four status tones appear only when the words have that state meaning. +Hierarchy starts with tone and weight before size — a secondary note usually +wants muted body or label, not a smaller custom font. If the prose needs +emphasis, improve the sentence before adding a style. + +The control emphasis ladder is exactly five rungs: primary, secondary, +outline, ghost, link. One primary action per view is the rule; everything +else steps down the ladder. Primary is for the action the screen exists to +complete — if two buttons both look primary, the hierarchy failed. +Destructive is a meaning, not a rung: it is rare, it names the destructive +act directly, and it is never borrowed for urgency, emphasis, or brand heat. + +Fields are quiet until they are active: hairline border, muted placeholder, +clear label, and a focused ring when the user engages. The focus ring is +guidance, not decoration. Errors state facts next to the field that caused +them — do not hide field errors in modals, toasts, or generic banners when +the user needs to fix one input. + +Balanced text is for headings and compact statements. Do not balance long +body copy into jagged reading. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.motion.md b/packages/ghost/src/init-payloads/skeleton/grammar.motion.md new file mode 100644 index 00000000..a2d95094 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.motion.md @@ -0,0 +1,24 @@ +--- +description: "Motion doctrine — motion is evidence of state change, never decoration; a closed vocabulary of three duration roles and one ease; gather for any transition, animation, or hover treatment." +--- + +Motion is evidence of a state change. It confirms hover, press, reveal, +collapse, entrance, exit, and spatial movement. It does not entertain. + +The entire vocabulary is three duration roles and one ease. Fast is for hover +and press. Normal is for reveals, fades, and small state changes. Slow is +reserved for spatial transitions where the user needs to understand movement. +Do not introduce novelty easings because a surface feels static; the ease's +character is a brand answer — see the temperature signature. + +Nothing loops except explicit loading states. A spinner may continue because +work continues. Decorative pulsing, floating, glowing, and attention-seeking +keyframes are off-language. + +Prefer opacity and small transform changes. If removing an animation does not +reduce comprehension, the animation was decoration. + +Condition: marketing and editorial surfaces may stage entrances — scroll +reveals and section transitions are part of editorial rhythm, still built +from the three durations and the one ease. In product UI the same staging is +decoration. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.rhythm.md b/packages/ghost/src/init-payloads/skeleton/grammar.rhythm.md new file mode 100644 index 00000000..172b72ef --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.rhythm.md @@ -0,0 +1,23 @@ +--- +description: "Layout rhythm — all layout is stacks with a closed gap step set; gather before laying anything out; never ad-hoc sibling margins." +--- + +All layout is stacks. Rhythm comes from relationships between siblings, not +from isolated margins pasted onto whichever element was last touched. + +The gap steps are exactly five: `--gap-xs`, `--gap-sm`, `--gap-md`, +`--gap-lg`, `--gap-xl`. Choose the gap that states the relationship. Do not +tune by single pixels to make a screenshot pass. + +The default is column, medium gap, stretch alignment, start justification. +That is the ordinary reading rhythm. + +Columns are for almost everything: forms, cards, message lists, modal bodies, +settings, empty states, and page sections. A column lets the user scan. + +Rows are conditional. Use them for controls, metadata lines, compact status, +and paired label/value moments. If a row starts wrapping awkwardly, it +probably wanted to be a column. + +Ad-hoc margins between siblings are forbidden because they hide the rhythm. +When spacing feels wrong, change the stack gap or split the stack. diff --git a/packages/ghost/src/init-payloads/skeleton/grammar.surfaces.md b/packages/ghost/src/init-payloads/skeleton/grammar.surfaces.md new file mode 100644 index 00000000..b9b4ce24 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/grammar.surfaces.md @@ -0,0 +1,26 @@ +--- +description: "Surface roles and the closed elevation set — gather for any card, popover, modal, dialog, scrim, or bordered container; flat is the default, borders are structural, exactly three elevation tiers." +--- + +Surface is the only way an element gets a background, border, radius, or +shadow. If a container needs visual treatment, it first needs a surface role. + +The vocabulary is closed: role, padding, radius, border, and elevation. These +axes create enough range for page flow, cards, popovers, muted blocks, +accents, and dark moments without inventing one-off boxes. + +The default surface is flat: no border, no shadow. We do not outline +everything to prove layout exists. + +The elevation tiers are exactly three: card, popover, modal. Elevation +implies hierarchy — a card sits in the document flow, a popover floats above +the flow, a modal interrupts the task. Pick the tier that matches the +interaction; never write a custom shadow because the composition feels flat, +and never pair a low tier with an interrupting role or give a routine card +modal gravity. Component shadows belong to the primitives that own them; +never borrow them for layout. + +Borders are structural, especially for inputs and overlays. They are not +decorative frames around ordinary text — reject borders as decoration, +especially around assistant text, and use space, tone, and type hierarchy +instead. diff --git a/packages/ghost/src/init-payloads/skeleton/index.md b/packages/ghost/src/init-payloads/skeleton/index.md new file mode 100644 index 00000000..303364f0 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/index.md @@ -0,0 +1,27 @@ +--- +description: "Always read first — the trust tiers of this starter fingerprint, the unanswered dials, and how to work before they are answered." +--- + +This is a skeleton fingerprint: the adaptation-safe law of good interface work, +with every brand decision left explicitly open. It steers an agent away from +the model's median from the first generation, without pretending to be a +brand it is not. + +The corpus carries two trust tiers. Grammar and the median floor are law: +safe to consume verbatim, unchanged by any adaptation — they speak in +token roles and closed sets, never in literal values. Signature nodes are +dials, and in this starter every dial is unanswered: each states the fixed +relationship worth keeping and the question only a human can answer. Do not +freehand a dial's value and present it as brand-backed — ask the human or +flag the gap. + +The reset raises the floor; only the fingerprint escapes the median. + +Unanswered dials do not block work; they mark what your report must flag. +When generating before the dials are answered, define the grammar's token +roles in your own stylesheet with provisional values — the role names are +the contract; the values are placeholders until the dials say otherwise. + +When a requested UI is not covered by a node, compose from the grammar. Do +not invent a new styling system. If the result needs a new pattern, the new +pattern should come from observed drift, not anticipation. diff --git a/packages/ghost/src/init-payloads/skeleton/signature.palette.md b/packages/ghost/src/init-payloads/skeleton/signature.palette.md new file mode 100644 index 00000000..530d8784 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/signature.palette.md @@ -0,0 +1,24 @@ +--- +description: "Gather whenever color beyond the base roles is in question. The palette dial: a quiet base spine plus a closed expression set at situation-gated volume — this dial is unanswered; ask the human or flag the gap, don't freehand it." +--- + +This dial is unanswered — ask the human or flag the gap. Do not freehand a +value and present it as brand-backed. + +The relationship is fixed: one brand at different volumes. A quiet base +spine is the default atmosphere in every medium — calm, legible, and +resistant to novelty — and a closed expression set supplies the atmosphere, +with volume set by situation, never by taste. Two constants outrank any +volume decision: expression never touches what you click — buttons, inputs, +and links stay on the base roles everywhere — and the status roles are not +expression; they keep their meanings everywhere and never moonlight as +atmosphere. The ladder — a quiet spine, a closed hue set, situation-gated +volume — is the part worth keeping. + +The open question: what is this brand's base spine, and what are the named +hues of its closed expression set — how many, and which? When the human +answers, restate this node as "this brand's current answer is …" and record +the values where your materials live. If the task can't wait for an answer, +proceed with a quiet provisional choice, honor the fixed relationship, label +the choice provisional in your report — and never restate this node as +answered. diff --git a/packages/ghost/src/init-payloads/skeleton/signature.shape.md b/packages/ghost/src/init-payloads/skeleton/signature.shape.md new file mode 100644 index 00000000..3f8ceb2d --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/signature.shape.md @@ -0,0 +1,19 @@ +--- +description: "Gather before setting any radius or corner treatment. The shape dial: controls take --radius-control, surfaces take --radius-surface — this dial is unanswered; ask the human or flag the gap, don't freehand it." +--- + +This dial is unanswered — ask the human or flag the gap. Do not freehand a +value and present it as brand-backed. + +The relationship is fixed: controls and surfaces carry different radius +roles, and the two never swap. Buttons and text inputs use +`--radius-control`; cards and other surfaces use `--radius-surface`. One +radius for what you click, one for what contains — that split is the part +worth keeping, whatever the values turn out to be. + +The open question: what are this brand's two radii — how round is a control, +and how round is a surface? When the human answers, restate this node as +"this brand's current answer is …" and record the values where your +materials live. If the task can't wait for an answer, proceed with a quiet +provisional choice, honor the fixed relationship, label the choice +provisional in your report — and never restate this node as answered. diff --git a/packages/ghost/src/init-payloads/skeleton/signature.temperature.md b/packages/ghost/src/init-payloads/skeleton/signature.temperature.md new file mode 100644 index 00000000..72ffad88 --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/signature.temperature.md @@ -0,0 +1,20 @@ +--- +description: "Gather for any copy and for motion character. The temperature dial: how the brand sounds and how it moves, one shared temperature — this dial is unanswered; ask the human or flag the gap, don't freehand it." +--- + +This dial is unanswered — ask the human or flag the gap. Do not freehand a +value and present it as brand-backed. + +The relationship is fixed: voice and motion carry the same temperature. Copy +states what happened, what is possible, or what the user must decide — it +does not perform personality — and motion confirms rather than entertains. +The coupling — words and motion sharing one temperature — is the part worth +keeping. + +The open question: where does this brand sit — how warm or cool is its copy, +what does its motion feel like, and what character does its one ease carry? +When the human answers, restate this node as "this brand's current answer +is …" and record the values where your materials live. If the task can't +wait for an answer, proceed with a quiet provisional choice, honor the fixed +relationship, label the choice provisional in your report — and never +restate this node as answered. diff --git a/packages/ghost/src/init-payloads/skeleton/signature.type.md b/packages/ghost/src/init-payloads/skeleton/signature.type.md new file mode 100644 index 00000000..e838028b --- /dev/null +++ b/packages/ghost/src/init-payloads/skeleton/signature.type.md @@ -0,0 +1,22 @@ +--- +description: "Gather for any text, and for heroes, landing pages, and editorial moments. The type dial: one voice typeface plus a mono for machine detail, editorial scale kept separate from product text — this dial is unanswered; ask the human or flag the gap, don't freehand it." +--- + +This dial is unanswered — ask the human or flag the gap. Do not freehand a +value and present it as brand-backed. + +The relationship is fixed: one typeface is the voice of the interface, +everywhere, with a mono variant only for code, tool detail, and machine +output. The heading scale is editorial — it exists for pages composed +outside the text variants, heroes and editorial moments — and product UI +never mixes the two vocabularies in one view. A modal title is not a poster. +The one-voice rule and the editorial/product split are the parts worth +keeping. + +The open question: what typeface is this brand's voice, what mono partners +it, and what rhythm does the editorial heading scale carry? When the human +answers, restate this node as "this brand's current answer is …" and record +the values where your materials live. If the task can't wait for an answer, +proceed with a quiet provisional choice, honor the fixed relationship, label +the choice provisional in your report — and never restate this node as +answered. diff --git a/packages/ghost/src/review/resolve.ts b/packages/ghost/src/review/resolve.ts index d7e79043..10c6c909 100644 --- a/packages/ghost/src/review/resolve.ts +++ b/packages/ghost/src/review/resolve.ts @@ -1,11 +1,12 @@ import { classifyMaterialLocator, type GhostCatalog, + type MaterialTransportOptions, + materialLocatorClaimsPath, parseSourceRef, } from "#ghost-core"; import type { LoadedCheck } from "../scan/check-files.js"; import { parseTouchedFiles, type TouchedFile } from "./diff.js"; -import { matchesGlob } from "./glob.js"; export interface MatchedMaterialNode { id: string; @@ -39,6 +40,7 @@ export function resolveReview( catalog: GhostCatalog, checks: Map, diffText: string, + transport: MaterialTransportOptions, ): ReviewResolution { const touchedFiles = parseTouchedFiles(diffText); const materialNodeIds = new Set(); @@ -55,8 +57,13 @@ export function resolveReview( if (localLocators.length === 0) continue; materialNodeIds.add(node.id); for (const file of touchedFiles) { + // Resolve each locator the same way validate does: package-relative + // `materials/…` locators expand to their repo-relative form before + // matching diff paths (which git emits repo-relative). Matching the + // raw locator text silently missed every exact-path locator whenever + // the package lives below the repo root. const locators = localLocators.filter((locator) => - matchesGlob(locator, file.path), + materialLocatorClaimsPath(locator, file.path, transport), ); if (locators.length === 0) continue; claimedFiles.add(file.path); diff --git a/packages/ghost/src/review/review-packet.ts b/packages/ghost/src/review/review-packet.ts index 9ed87017..34f72895 100644 --- a/packages/ghost/src/review/review-packet.ts +++ b/packages/ghost/src/review/review-packet.ts @@ -1,5 +1,8 @@ +import { join } from "node:path"; import type { GhostCatalogNode } from "#ghost-core"; +import { GHOST_MATERIALS_DIR } from "../scan/constants.js"; import type { LoadedFingerprintPackage } from "../scan/fingerprint-package.js"; +import { resolveGitRoot } from "../scan/package-paths.js"; import { type BaselineProse, resolveBaseline } from "./baseline.js"; import { type ProbeEvidence, runProbe } from "./probes.js"; import type { CoverageGap } from "./resolve.js"; @@ -38,6 +41,8 @@ export interface ReviewPacket { } export interface BuildReviewPacketOptions { + /** Absolute path of the fingerprint package directory (default: cwd/.ghost). */ + packageDir?: string; runProbes?: boolean; cwd?: string; probeTimeoutMs?: number; @@ -48,10 +53,16 @@ export async function buildReviewPacket( diffText: string, options: BuildReviewPacketOptions = {}, ): Promise { + const cwd = options.cwd ?? process.cwd(); const resolution = resolveReview( fingerprint.catalog, fingerprint.checks, diffText, + { + repoRoot: await resolveGitRoot(cwd), + packageDir: options.packageDir ?? join(cwd, ".ghost"), + materialsDir: GHOST_MATERIALS_DIR, + }, ); const materialNodes: PacketMaterialNode[] = resolution.materialNodes.map( diff --git a/packages/ghost/src/scan/check-scaffold.ts b/packages/ghost/src/scan/check-scaffold.ts index c1c5073a..4e23f9d9 100644 --- a/packages/ghost/src/scan/check-scaffold.ts +++ b/packages/ghost/src/scan/check-scaffold.ts @@ -2,8 +2,10 @@ import { access, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { UsageError } from "#ghost-core"; import { GHOST_CHECKS_DIR } from "./check-files.js"; +import { loadPayloadFile } from "./packed-payloads.js"; const EXAMPLE_CHECK_FILENAME = "example.md.example"; +const MEDIAN_TELLS_FILENAME = "median-tells.md"; const EXAMPLE_CHECK_CONTENT = `--- name: logo-clearspace-holds @@ -21,6 +23,7 @@ is used when the full lockup is required. export interface AddChecksResult { dir: string; written: string[]; + skipped: string[]; } /** Scaffold the flat `.ghost/checks/` directory with an example check. */ @@ -32,13 +35,33 @@ export async function addChecksDir( throw new UsageError(`checks/ already exists at ${checksDir}.`); } + const written: string[] = []; + const skipped: string[] = []; + await mkdir(checksDir, { recursive: true }); + if (await exists(join(packageDir, "anti-goal.median.md"))) { + await writeFile( + join(checksDir, MEDIAN_TELLS_FILENAME), + await loadPayloadFile("median", MEDIAN_TELLS_FILENAME), + "utf-8", + ); + written.push(MEDIAN_TELLS_FILENAME); + } else { + skipped.push(`${MEDIAN_TELLS_FILENAME} (no anti-goal.median node)`); + } + await writeFile( join(checksDir, EXAMPLE_CHECK_FILENAME), EXAMPLE_CHECK_CONTENT, "utf-8", ); - return { dir: checksDir, written: [EXAMPLE_CHECK_FILENAME] }; + written.push(EXAMPLE_CHECK_FILENAME); + + return { + dir: checksDir, + written, + skipped, + }; } async function exists(path: string): Promise { diff --git a/packages/ghost/src/scan/fingerprint-package.ts b/packages/ghost/src/scan/fingerprint-package.ts index 6d8de169..2255e023 100644 --- a/packages/ghost/src/scan/fingerprint-package.ts +++ b/packages/ghost/src/scan/fingerprint-package.ts @@ -33,7 +33,10 @@ import type { LintIssue, LintReport } from "./lint.js"; import { resolveGitRoot } from "./package-paths.js"; import { DEFAULT_TEMPLATE_NAME, + type GhostInitTemplate, + getInitBody, getInitTemplate, + listInitBodies, listInitTemplates, } from "./templates.js"; @@ -70,8 +73,14 @@ export interface LoadedFingerprintPackage { } export interface InitFingerprintPackageOptions { - /** Init template name (default: "steering"). */ + /** Init template name (default: "skeleton"). Mutually exclusive with `body`. */ template?: string; + /** + * Init body name (e.g. "vessel-light"): a full inhabited package with + * answered dials, materials, and its own checks. Mutually exclusive with + * `template`. + */ + body?: string; force?: boolean; } @@ -103,18 +112,36 @@ export async function initFingerprintPackage( cwd = process.cwd(), options: InitFingerprintPackageOptions = {}, ): Promise { - const templateName = options.template ?? DEFAULT_TEMPLATE_NAME; - const template = getInitTemplate(templateName); - if (!template) { + if (options.body !== undefined && options.template !== undefined) { throw new UsageError( - `Unknown init template '${templateName}'. Available: ${listInitTemplates().join(", ")}.`, + "--body and --template are mutually exclusive. A template is a shape of emptiness; a body is a full inhabited package — pick one.", ); } + let source: Pick; + if (options.body !== undefined) { + const body = getInitBody(options.body); + if (!body) { + throw new UsageError( + `Unknown init body '${options.body}'. Available: ${listInitBodies().join(", ")}.`, + ); + } + source = body; + } else { + const templateName = options.template ?? DEFAULT_TEMPLATE_NAME; + const template = getInitTemplate(templateName); + if (!template) { + throw new UsageError( + `Unknown init template '${templateName}'. Available: ${listInitTemplates().join(", ")}.`, + ); + } + source = template; + } + const paths = resolveFingerprintPackage(dirArg, cwd); await mkdir(paths.packageDir, { recursive: true }); - const files = template.files().map((file) => ({ + const files = (await source.files()).map((file) => ({ relativePath: file.relativePath, path: join(paths.packageDir, file.relativePath), content: file.content, @@ -137,14 +164,17 @@ export async function initFingerprintPackage( async function writeInitFile( path: string, - content: string, + content: string | Uint8Array, force = false, ): Promise { try { - await writeFile(path, content, { - encoding: "utf-8", - flag: force ? "w" : "wx", - }); + await writeFile( + path, + content, + typeof content === "string" + ? { encoding: "utf-8", flag: force ? "w" : "wx" } + : { flag: force ? "w" : "wx" }, + ); } catch (err) { if (!force && isExistingPathError(err)) { throw new UsageError( @@ -395,7 +425,7 @@ function lintCheckReferences( issues.push({ severity: "warning", rule: "check-reference-unresolved", - message: `check reference '${raw}' does not resolve to a fingerprint node`, + message: `check reference '${raw}' does not resolve to a fingerprint node — if you pruned this rule from the node, delete its paired flag in the check too`, path: `checks/${check.id}.md.references`, }); continue; @@ -407,7 +437,7 @@ function lintCheckReferences( issues.push({ severity: "warning", rule: "check-reference-heading-missing", - message: `check reference '${raw}' names a heading that was not found`, + message: `check reference '${raw}' names a heading that was not found — if you pruned this rule from the node, delete its paired flag in the check too`, path: `checks/${check.id}.md.references`, }); } diff --git a/packages/ghost/src/scan/packed-payloads.ts b/packages/ghost/src/scan/packed-payloads.ts new file mode 100644 index 00000000..2c7142cb --- /dev/null +++ b/packages/ghost/src/scan/packed-payloads.ts @@ -0,0 +1,72 @@ +import { existsSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { TemplateFile } from "./templates.js"; + +/** + * Payload roots, first match wins. At runtime this module lives in + * `dist/scan/`, so `../init-payloads` is the packed payload dir. Under + * vitest it runs from `src/scan/`, where only committed payloads (skeleton) + * exist — synced payloads (vessel-light) resolve through the built + * `dist/init-payloads` sibling. + */ +const INIT_PAYLOAD_ROOTS = [ + fileURLToPath(new URL("../init-payloads", import.meta.url)), + fileURLToPath(new URL("../../dist/init-payloads", import.meta.url)), +]; + +const BINARY_EXTENSIONS = new Set([".woff", ".woff2"]); + +export async function loadPackedPayload(name: string): Promise { + const payloadDir = resolvePayloadDir(name); + const files = await listPayloadFiles(payloadDir); + + return Promise.all( + files.map(async (path) => ({ + relativePath: relative(payloadDir, path), + content: await readPayloadFile(path), + })), + ); +} + +export async function loadPayloadFile( + payload: string, + relativePath: string, +): Promise { + return readFile(join(resolvePayloadDir(payload), relativePath), "utf-8"); +} + +function resolvePayloadDir(name: string): string { + return ( + INIT_PAYLOAD_ROOTS.map((root) => join(root, name)).find((dir) => + existsSync(dir), + ) ?? join(INIT_PAYLOAD_ROOTS[0], name) + ); +} + +async function listPayloadFiles(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const path = join(dir, entry.name); + if (entry.isDirectory()) return listPayloadFiles(path); + if (entry.isFile()) return [path]; + return []; + }), + ); + + return files.flat().sort((a, b) => a.localeCompare(b)); +} + +async function readPayloadFile(path: string): Promise { + const bytes = await readFile(path); + if (isBinaryPayloadPath(path)) return bytes; + return bytes.toString("utf-8"); +} + +function isBinaryPayloadPath(path: string): boolean { + const dot = path.lastIndexOf("."); + if (dot === -1) return false; + return BINARY_EXTENSIONS.has(path.slice(dot).toLowerCase()); +} diff --git a/packages/ghost/src/scan/steering-template.ts b/packages/ghost/src/scan/steering-template.ts deleted file mode 100644 index a9441f4e..00000000 --- a/packages/ghost/src/scan/steering-template.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { GhostInitTemplate, TemplateFile } from "./templates.js"; - -interface SteeringTemplateDeps { - manifestFile(): TemplateFile; - gitignoreFile(): TemplateFile; -} - -export function createSteeringTemplate({ - manifestFile, - gitignoreFile, -}: SteeringTemplateDeps): GhostInitTemplate { - return STEERING_TEMPLATE_FACTORY(manifestFile, gitignoreFile); -} - -/** - * Steering starter: named truths for pre-generation steering. The extra kinds - * are vocabulary only: no schema or selection behavior changes. - */ -function STEERING_TEMPLATE_FACTORY( - manifestFile: () => TemplateFile, - gitignoreFile: () => TemplateFile, -): GhostInitTemplate { - return { - name: "steering", - description: "Steering starter: stance, patterns, exemplars, materials.", - files() { - return [ - manifestFile(), - gitignoreFile(), - { - relativePath: "glossary.md", - content: `--- -kinds: - - name: principle - - name: condition - - name: pattern - - name: exemplar - - name: anti-goal - posture: guard - - name: asset - - name: decision - - name: concept -# - { name: provocation, posture: wild, purpose: a deliberate provocation past the fingerprint — surfaced only on request } ---- - -# principle - -Durable stance, tradeoff, or invariant. Patterns may narrow principles, never loosen them. - -# condition - -Situational truth. Use only when the stated situation holds. - -# pattern - -Reusable composition. State what is bound, what is open, and when it applies. - -# exemplar - -Concrete reference for form, rhythm, quality, code shape, voice, or behavior. Say what to copy and not copy. - -# anti-goal - -Review-critical guard posture. State not-X-instead-Y: the rejected default, the positive replacement, and how to recognize the switch. - -# asset - -Material truth about concrete assets, tokens, components, type, motion, imagery, code, or files. \`materials\` locates; prose explains. - -# decision - -Worked tradeoff trace: what was chosen, what was rejected, and why. - -# concept - -Scoped creative direction for a specific output; not permanent brand law. -`, - }, - { - relativePath: "index.md", - content: `--- -description: Always read first — the steering front door, non-negotiables, and silence posture. ---- - -This fingerprint is demo content for **Morrow Ledger**, a small bookkeeping product for independent studios. It steers agent-generated product and brand work; it is not a component API or complete archive. - -## Before you trust this fingerprint - -Everything in this starter package is demo content. It is intentionally opinionated so you can see the shape of useful steering: hard truths, refusal lines, material locators, patterns, exemplars, and tradeoff traces. Keep any structure that helps, but replace the claims, paths, examples, and decisions with real product truth before using this fingerprint to guide generation. - -## Non-negotiables - -The lines below are demo non-negotiables for inspiration and guidance. Replace them with your real hard truths: - -- Morrow Ledger sounds like a careful bookkeeper, not a growth coach. -- Use slate ink, ledger green, and quiet amber before inventing new color. -- Preserve the composition floor in \`principle.composition\`. -- Avoid the generic generated defaults named in \`anti-goal.*\`. -- Label provisional reasoning when this fingerprint is silent. - -## How to read the corpus - -\`principle.*\` gives stance and hard floor. \`pattern.*\` gives bound/open structure. \`exemplar.*\` shows form and quality. \`anti-goal.*\` names likely wrong outputs. \`asset.*\` points to materials. \`decision.*\` explains tradeoffs. \`concept.*\` is scoped direction, not law. The starter nodes are teaching examples, not defaults Ghost believes about your brand. - -## Silence posture - -When this fingerprint is silent, proceed provisionally from local conventions only when safe. Ask before brand-defining, irreversible, legal, privacy-sensitive, or high-risk choices. -`, - }, - { - relativePath: "principle.stance.md", - content: `--- -description: Core stance and tradeoffs — gather when a task needs brand decision-making beyond exact materials. ---- - -Morrow Ledger helps independent studios know what changed in their money without making finance feel like a performance dashboard. - -Forced choices: - -- **Clarity beats momentum.** When a line could sound exciting or precise, choose precise. Write "3 invoices are 14+ days late" instead of "Cash flow needs attention." -- **Evidence beats reassurance.** Pair every claim with the number, date, file, or account behind it. Never say "all set" unless the surface shows what was checked. -- **Calm beats clever.** Use plain verbs: sent, due, matched, missing, paid. Avoid pep-talk verbs: crush, unlock, supercharge. -- **Small business time is scarce.** Lead with the next accountable action and its cost: "Review 2 unmatched deposits — about 4 minutes." - -The brand earns trust by making the boring truth easy to act on. If the work feels motivational, it is drifting. - -`, - }, - { - relativePath: "principle.composition.md", - content: `--- -description: Composition floor — hard layout invariants that hold whether or not a pattern matches the ask. ---- - -These demo invariants show the kind of floor that steers generation. Replace them with real hard lines that apply to every generated surface unless a narrower condition explicitly says otherwise. Patterns narrow them; nothing loosens them. - -For Morrow Ledger: - -- **One ledger fact leads.** Every surface opens with one factual sentence or number. If two facts compete, choose the one that changes what the user does next. -- **Maximum three figures above the fold.** A figure is any amount, count, date, or percentage. If there are four, demote one into details instead of shrinking type. -- **Actions are accountable verbs.** Primary actions begin with Review, Match, Send, Export, or Mark. Never use generic "Continue" when the action can be named. -- **Whitespace before boxes.** Separate sections with 24px vertical space first, a 1px #D8DED8 rule second, and a card only when the content has its own state. - -Token sketch for demos: - -\`\`\`css -:root { - --morrow-ink: #17201b; - --morrow-ledger: #2f6f4f; - --morrow-amber: #b7791f; - --morrow-paper: #f7f5ef; -} -\`\`\` - -`, - }, - { - relativePath: "anti-goal.generic-generated-output.md", - content: `--- -description: The generic generated output this fingerprint rejects. ---- - -This is demo anti-goal content. Replace it with the defaults this brand actually refuses. - -Not: a rounded-card SaaS dashboard full of gradients, celebratory deltas, and "You're crushing it" copy. - -Instead: a ledger-first surface with one leading fact, muted paper background, accountable next action, and evidence close to the claim. - -Concrete replacements: - -- Not "Revenue is up 24% 🎉" — instead "July receipts are $4,820 higher than June; 2 invoices explain most of the change." -- Not a decorative blob or fake chart preview — instead a compact list of the source transactions or invoices. -- Not "Oops, something went wrong" — instead "We could not match deposit 1042. Choose the invoice or mark it as owner contribution." - -A technically correct output can still be off-brand if any product could ship it. This guard is review-critical because generic finance optimism is the most likely model default. -`, - }, - { - relativePath: "pattern.status-with-next-step.md", - content: `--- -description: Status of an ongoing thing plus the single next step — use when the task asks where something stands or what happens next. ---- - -This is a demo pattern for inspiration and guidance. Replace it with a real reusable pattern from your product. - -Applies when the surface explains the state of something in flight: invoice payment, bank sync, export, tax packet, or reconciliation. - -**Bound (do not redecide):** - -- Status line first, exactly one sentence, no more than 11 words. -- One evidence row with amount, date, and source. -- One primary next step last, beginning with an accountable verb. - -**Open (within limits):** - -- Evidence may be a table row or two-item list. -- Use amber only for a user action needed within 7 days. - -## Skeleton - -\`\`\`tsx -
-

{scope}

-

{statusSentence}

-

{meaningSentence}

-
{amountDateSource}
- -
-\`\`\` - -`, - }, - { - relativePath: "exemplar.annotated-reference.md", - content: `--- -description: Annotated reference — replace with a sample that shows form, rhythm, quality, voice, or code shape. -materials: - - src/components/status-card.tsx ---- - -This is demo exemplar content. Replace \`src/components/status-card.tsx\` with a real screenshot, file, URL, or implementation path. - -Normative for: density, evidence placement, accountable action language. - -Annotated reference: - -\`\`\`tsx - -\`\`\` - -Copy the square edge, measured evidence, and action cost. Do not copy the exact amounts or dates. -`, - }, - { - relativePath: "asset.materials.md", - content: `--- -description: Concrete materials the agent should inspect before inventing visual language, components, type, motion, or assets. -materials: - - src/styles/tokens.css - - src/components/** - - public/brand/** ---- - -This is demo material inventory. Replace the paths with real tokens, components, SVGs, motion specs, images, or implementations. - -Morrow Ledger's starter materials should resolve to real files in your repo before use: - -- \`src/styles/tokens.css\` should define the ink, ledger, amber, paper, and rule colors. -- \`src/components/**\` should include the production Button, InlineAlert, and DataRow components. -- \`public/brand/**\` should include the wordmark and any blessed product screenshots. - -Use this node to locate real materials. \`materials\` points to concrete files; this prose explains what they mean. Do not duplicate API docs unless the API itself is the brand truth. -`, - }, - { - relativePath: "decision.tradeoff.md", - content: `--- -description: Worked tradeoff — replace with a decision where the brand chose between plausible options. ---- - -This is demo decision content. Replace it with a real tradeoff that helps an agent decide between plausible good answers. - -Morrow Ledger tradeoff: we chose visible evidence over shorter surfaces. - -- Option A: a compact success banner, "Bank sync complete." It was calm and fast, but it hid what changed. -- Option B: a slightly taller status block, "Bank sync complete; 14 transactions imported since Jul 12." It costs one extra line but gives the user a fact to trust. -- Decision: choose Option B whenever the action touches money, taxes, invoices, exports, or bank data. -- Reverses when: the evidence would expose sensitive third-party details in a shared view. Then show the count only and link to details. -`, - }, - ]; - }, - }; -} diff --git a/packages/ghost/src/scan/templates.ts b/packages/ghost/src/scan/templates.ts index 92c53446..8de78dac 100644 --- a/packages/ghost/src/scan/templates.ts +++ b/packages/ghost/src/scan/templates.ts @@ -3,14 +3,14 @@ import { GHOST_EVENTS_FILENAME, LEGACY_PULL_HISTORY_FILENAME, } from "./constants.js"; -import { createSteeringTemplate } from "./steering-template.js"; +import { loadPackedPayload, loadPayloadFile } from "./packed-payloads.js"; /** * A single seed file an `init` template writes, relative to the package dir. */ export interface TemplateFile { /** Path relative to the package directory (e.g. "principle.voice.md"). */ relativePath: string; - content: string; + content: string | Uint8Array; } /** * An `init` template: a pure description of the seed files a fresh node package @@ -21,7 +21,7 @@ export interface TemplateFile { export interface GhostInitTemplate { name: string; description: string; - files(): TemplateFile[]; + files(): TemplateFile[] | Promise; } function manifestFile(): TemplateFile { return { @@ -52,7 +52,8 @@ const MINIMAL_TEMPLATE: GhostInitTemplate = { name: "minimal", description: "Minimal node package: manifest + glossary + a starter index node.", - files() { + async files() { + const medianFile = await medianTemplateFile(); return [ manifestFile(), gitignoreFile(), @@ -87,6 +88,8 @@ evidence unless the node says the sample itself is normative. What this brand must never look, sound, or feel like — named generic patterns and rejected neighbors. Always-on, like a principle, but stated as the thing to steer away from. +\`anti-goal.median\` is the model's floor, not the brand's taste. Gather +anti-goals before styling anything greenfield. # asset @@ -127,6 +130,7 @@ When a truth is narrower, state the condition in the prose — the situation whe it applies — never a filing destination. `, }, + medianFile, ]; }, }; @@ -146,7 +150,8 @@ const COMPOSITION_TEMPLATE: GhostInitTemplate = { name: "composition", description: "Composition starter: minimal files + an invariants floor and a worked bound/open pattern.", - files() { + async files() { + const medianFile = await medianTemplateFile(); return [ manifestFile(), gitignoreFile(), @@ -185,6 +190,8 @@ render travels with the prose. What this brand must never look, sound, or feel like — named generic patterns and rejected neighbors. Always-on, like a principle, but stated as the thing to steer away from. +\`anti-goal.median\` is the model's floor, not the brand's taste. Gather +anti-goals before styling anything greenfield. # asset @@ -297,26 +304,125 @@ When a blessed render of this pattern exists, add an \`exemplar.*\` node with \`materials\` pointing at the screenshot and the implementation path. `, }, + medianFile, ]; }, }; -const STEERING_TEMPLATE = createSteeringTemplate({ - manifestFile, - gitignoreFile, -}); +const SKELETON_FILE_ORDER = new Map( + [ + "glossary.md", + "index.md", + "anti-goal.median.md", + "grammar.hierarchy.md", + "grammar.rhythm.md", + "grammar.surfaces.md", + "grammar.motion.md", + "grammar.color-roles.md", + "grammar.conversation.md", + "signature.shape.md", + "signature.palette.md", + "signature.type.md", + "signature.temperature.md", + ].map((path, index) => [path, index]), +); + +const SKELETON_TEMPLATE: GhostInitTemplate = { + name: "skeleton", + description: + "Naked skeleton: the median floor + grammar law, with the signature dials left unanswered.", + async files() { + const skeletonFiles = [ + ...(await loadPackedPayload("skeleton")), + await medianTemplateFile(), + ]; + skeletonFiles.sort( + (a, b) => + (SKELETON_FILE_ORDER.get(a.relativePath) ?? Number.MAX_SAFE_INTEGER) - + (SKELETON_FILE_ORDER.get(b.relativePath) ?? + Number.MAX_SAFE_INTEGER) || + a.relativePath.localeCompare(b.relativePath), + ); + return [manifestFile(), gitignoreFile(), ...skeletonFiles]; + }, +}; + +async function medianTemplateFile(): Promise { + return { + relativePath: "anti-goal.median.md", + content: await loadPayloadFile("median", "anti-goal.median.md"), + }; +} const TEMPLATES = new Map([ [MINIMAL_TEMPLATE.name, MINIMAL_TEMPLATE], [COMPOSITION_TEMPLATE.name, COMPOSITION_TEMPLATE], - [STEERING_TEMPLATE.name, STEERING_TEMPLATE], + [SKELETON_TEMPLATE.name, SKELETON_TEMPLATE], ]); -export const DEFAULT_TEMPLATE_NAME = STEERING_TEMPLATE.name; +/** + * An init body: a full inhabited fingerprint package — answered signature + * dials, materials, refs, and its own checks. Templates are shapes of + * emptiness awaiting the owner's truths; a body is the same anatomy with a + * real brand's values plugged in. Bodies keep their own manifest id (e.g. + * `vessel-light`) so an unadapted install stays honestly labeled — changing + * the id is step one of adapting the starter, an explicit human act. + */ +export interface GhostInitBody { + name: string; + description: string; + /** Whether the body payload ships its own `checks/` directory. */ + includesChecks: boolean; + files(): Promise; +} + +const VESSEL_LIGHT_BODY: GhostInitBody = { + name: "vessel-light", + description: + "Vessel's design language at full strength: corpus, tokens, fonts, refs, and checks.", + includesChecks: true, + async files() { + const payload = await loadPackedPayload("vessel-light"); + payload.sort((a, b) => { + const ao = BODY_FILE_ORDER.get(a.relativePath); + const bo = BODY_FILE_ORDER.get(b.relativePath); + return ( + (ao ?? Number.MAX_SAFE_INTEGER) - (bo ?? Number.MAX_SAFE_INTEGER) || + a.relativePath.localeCompare(b.relativePath) + ); + }); + // The payload carries its own manifest.yml (id stays `vessel-light`); + // only the gitignore is generated locally. + return [gitignoreFile(), ...payload]; + }, +}; + +/** Listing order for the body scaffold output: anchors first, then the rest. */ +const BODY_FILE_ORDER = new Map( + ["manifest.yml", "glossary.md", "index.md"].map((path, i) => [path, i]), +); + +const BODIES = new Map([ + [VESSEL_LIGHT_BODY.name, VESSEL_LIGHT_BODY], +]); + +/** Look up a registered init body by name. */ +export function getInitBody(name: string): GhostInitBody | undefined { + return BODIES.get(name); +} + +export function listInitBodies(): string[] { + return [...BODIES.keys()]; +} + +export const DEFAULT_TEMPLATE_NAME = SKELETON_TEMPLATE.name; /** Look up a registered init template by name. */ export function getInitTemplate(name: string): GhostInitTemplate | undefined { - if (name === "default") return STEERING_TEMPLATE; + // `default` and `steering` are silent aliases for the skeleton starter: + // `default` for callers that never named a template, `steering` for + // callers pinned to the pre-skeleton default's name. + if (name === "default" || name === "steering") return SKELETON_TEMPLATE; return TEMPLATES.get(name); } diff --git a/packages/ghost/src/skill-bundle/SKILL.md b/packages/ghost/src/skill-bundle/SKILL.md index 414bfcc0..a54843df 100644 --- a/packages/ghost/src/skill-bundle/SKILL.md +++ b/packages/ghost/src/skill-bundle/SKILL.md @@ -110,6 +110,7 @@ against the unpacked package with `--package `. - Author or update the fingerprint: follow [references/capture.md](references/capture.md). - Author material-backed nodes: follow [references/blocks.md](references/blocks.md). - Choose the right human-agent authoring workflow: follow [references/authoring-scenarios.md](references/authoring-scenarios.md). +- Adapt a starter (a body or the skeleton) to your brand: follow [references/adapting-a-starter.md](references/adapting-a-starter.md). - Gather applicable truths for a task: follow [references/recall.md](references/recall.md). - Shape a pre-generation brief: follow [references/brief.md](references/brief.md). - Explore deliberate provocations with wild nodes: follow [references/wild.md](references/wild.md). diff --git a/packages/ghost/src/skill-bundle/references/adapting-a-starter.md b/packages/ghost/src/skill-bundle/references/adapting-a-starter.md new file mode 100644 index 00000000..31792f16 --- /dev/null +++ b/packages/ghost/src/skill-bundle/references/adapting-a-starter.md @@ -0,0 +1,102 @@ +--- +name: adapting-a-starter +description: Transplant a starter fingerprint — a body like vessel-light or the naked skeleton — into your own brand, one procedure from manifest id to regenerated refs. +handoffs: + - label: Validate the adapted package + command: ghost validate --format json + prompt: Does the adapted fingerprint validate, and did any pruned rule orphan a check reference? +--- + +# Recipe: Adapt A Starter Fingerprint + +**Goal:** turn an installed starter — a full body (`ghost init --body +vessel-light`) or the naked skeleton (`ghost init`) — into *your* brand's +fingerprint without shipping a self-contradicting package. + +A starter is factored by rate of change under adaptation. Knowing which stratum a +file belongs to tells you what to do with it: + +| Stratum | Files | On adaptation | +| --- | --- | --- | +| Grammar | `grammar.*` | Keep unchanged — value-free decision logic that survives any adaptation. | +| Median floor | `anti-goal.median` | Prune, never rewrite — Ghost stamps this measured model truth into every initialized package; you own it after init. | +| Signature | `signature.*` | Answer — each is a dial; restate it with your brand's answer. | +| Values | `materials/tokens.css` | Edit — the single injection point for every literal value. | +| Registers | `register.*` | Re-tune — conditions referencing signature ids; revisit after the dials change. | +| Derived artifacts | `materials/ref/*.html`, `anti-goal.tells` | Regenerate — they demonstrate the values and near-misses of a *specific* signature. | + +Do the steps **in order and in one sitting** where possible. A half-adapted +package is worse than an unadapted one: stale refs steer harder than any prose +you rewrote, so stopping after step 4 ships a fingerprint that contradicts +itself. + +## The procedure + +1. **Change the manifest id.** Edit `id:` in `manifest.yml` to your brand's + name. This is deliberately first: it is the explicit act that marks the + adaptation as begun. Until it changes, the package honestly claims to be the + starter, and every consuming agent cites it as a starter default. +2. **Prune `anti-goal.median`.** Each rule is a `###` heading section; delete + the whole section for every rule your brand legitimately violates (a brand + built on gradients deletes the Gradients section — that is the node working, + not failing). Do not rewrite surviving rules; they are the model's measured + floor, not your taste. Then run `ghost validate`: every check reference + orphaned by a pruned heading surfaces as its own warning — delete the paired + flag and its reference from the check. +3. **Answer the signature dials.** Walk each `signature.*` node as a + questionnaire item. Keep the fixed relationship (the part the node marks + as worth keeping); replace the starter's answer — or the open question — + with your brand's: restate the node as "this brand's current answer is …". + Ask the human for any dial they have not decided; never freehand a value + and present it as brand-backed. +4. **Edit `materials/tokens.css`.** Every literal value lives here — radii, + palette, type sizes, durations, eases. Change the values; keep the role + names. The role names are the grammar's vocabulary and the reason the + grammar nodes survive untouched. +5. **Regenerate the refs.** This is the step that decides whether the adaptation + succeeded. Exemplars dominate prose: a prose rule contradicted by a stale + ref loses. Rebuild each `materials/ref/*.html` against the new tokens and + answered dials, keep the annotation headers (`normative-for` / + `incidental`) current, and make each ref demonstrate its closed sets + completely. If the starter shipped no refs (the skeleton), generate them + now — a fingerprint with no exemplars steers at half strength. +6. **Rewrite `anti-goal.tells`.** The tells are near-misses of the *starter's* + signature; yours are different. For each answered dial, name the failure + mode one step away from your answer and its replacement. If the starter + shipped no tells node (the skeleton), author one. +7. **Re-run the checks — including against the refs.** `ghost validate` for + package shape, then review the regenerated refs against the median and + value checks (stage the ref changes and run `ghost review`). The floor + only holds if the refs hold it too: a ref that trips a median flag will + teach every future generation the violation. Rewrite the body's + `checks/values.md` alongside the dials it references. + +## Consuming an unadapted starter + +Work does not block on adaptation. Before the procedure runs (or midway +through it), cite starter content honestly: + +- Grammar: **Ghost-backed** — value-free decision logic that holds for any + brand. +- Surviving median rules: **owner-backed after init** — Ghost stamps this + measured model truth into every initialized package; you own the pruning and + any adaptation thereafter. +- The starter's signature values (a body) or your provisional choices (the + skeleton): **Ghost-backed (starter default, unadapted)** or **provisional** + — never plain brand truth. The manifest id tells you which state you are + in: a starter id means unadapted. +- In a brief, `anti-goal.median` sits in the anti-goals slot — after intent, + inventory, and composition, never before the brand truths. If a median rule + conflicts with an answered signature node, the signature wins and the + median line is a prune candidate to report. + +## Never + +- Never rewrite grammar nodes to taste — if a grammar rule is wrong for your + brand, it was never grammar; move it to a signature node and answer it. +- Never leave the starter's refs alongside your new tokens — regenerate or + delete; a stale exemplar outweighs your rewritten prose. +- Never prune a median rule without deleting its paired check flag, or keep a + check that asserts an obligation no node states. +- Never pre-write the manifest id change into automation — it is the human's + act of ownership, the one step an agent should not take alone. diff --git a/packages/ghost/src/skill-bundle/references/authoring-scenarios.md b/packages/ghost/src/skill-bundle/references/authoring-scenarios.md index db308fb8..3bef7fb1 100644 --- a/packages/ghost/src/skill-bundle/references/authoring-scenarios.md +++ b/packages/ghost/src/skill-bundle/references/authoring-scenarios.md @@ -86,7 +86,7 @@ and enforced in review, not repeated as the model's main example. ## 4. Draft The Nodes Write the smallest useful set of nodes, each a purpose-coherent prose truth with -a one-line `description`, named `..md` (or a bare slug when no kind is present). Ask three questions of each body: why (the stance), with what +a one-line `description`, named `..md` (or a bare slug when no kind is present). Ask three questions of each node body: why (the stance), with what (the materials), and how it is assembled (the patterns). These are drafting prompts, not fields. @@ -134,6 +134,45 @@ prefix is a declared glossary kind (undeclared → warning with a "did you mean" suggestion). Use ordinary Git review as the approval boundary: uncommitted edits are drafts; checked-in nodes are canonical. +## Shapes Of A Useful Node + +The shapes below teach what steering prose looks like when it works. The +examples use **Morrow Ledger**, a fictional bookkeeping product — teaching +fiction for this recipe only, never content for a real corpus. Each shape is a +pattern to imitate with the human's real truths. + +- **Stance as forced choices.** Not adjectives — decisions under pressure: + "Clarity beats momentum: write '3 invoices are 14+ days late', not 'Cash + flow needs attention.'" A stance earns its place when it rejects a plausible + wrong answer. +- **Composition floor.** Hard invariants with numbers, holding unless a + narrower condition says otherwise: "Maximum three figures above the fold — + if there are four, demote one into details instead of shrinking type." +- **Anti-goal as not-X-instead-Y.** The rejected default *and* its committed + replacement: "Not 'Revenue is up 24% 🎉' — instead 'July receipts are $4,820 + higher than June; 2 invoices explain most of the change.'" A bare ban prunes + one path; the replacement commits another. +- **Pattern as bound/open.** Say what is decided and what remains free: + "Bound: status line first, one sentence, ≤11 words; one primary next step + last. Open: evidence may be a table row or a two-item list." All bound is a + template; nothing bound is vibes. +- **Annotated exemplar.** A concrete reference plus what it is normative for + and what is incidental: "Copy the square edge, measured evidence, and action + cost. Do not copy the exact amounts or dates." An unannotated exemplar + teaches its accidents as brand. +- **Material inventory.** `materials` locates; prose explains meaning: + "`src/styles/tokens.css` should define the ink, ledger, amber, and paper + colors." Never duplicate API docs unless the API itself is the brand truth. +- **Decision trace.** A worked tradeoff with the reversal condition: "We chose + visible evidence over shorter surfaces… Reverses when the evidence would + expose sensitive third-party details — then show the count only." Models + imitate reasoning as well as outputs. + +State a silence posture somewhere the agent always reads (usually `index.md`): +when the fingerprint is silent, proceed provisionally from local conventions +only when safe, and ask before brand-defining, irreversible, legal, +privacy-sensitive, or high-risk choices. + ## Never - Never derive brand truth from repo code alone; what the codebase repeats may diff --git a/packages/ghost/src/skill-bundle/references/blocks.md b/packages/ghost/src/skill-bundle/references/blocks.md index b6a2df40..6d6f31e0 100644 --- a/packages/ghost/src/skill-bundle/references/blocks.md +++ b/packages/ghost/src/skill-bundle/references/blocks.md @@ -57,7 +57,7 @@ Neither is correct. A concrete block node is a deliberate trade, not a leak. reasoning, sources…) earns **one short prose body**. This is what the method is for. - The **composer middle** (card, table, form, sidebar…) is a call to weigh. Give - it a body when its arrangement carries a stance worth matching. + it a prose body when its arrangement carries a stance worth matching. If a primitive ever seems to need stance guidance, that is a signal it is doing a composer's job. Promote the pattern into a node; do not write a body on the diff --git a/packages/ghost/src/skill-bundle/references/capture.md b/packages/ghost/src/skill-bundle/references/capture.md index 2e4324fd..76655ebb 100644 --- a/packages/ghost/src/skill-bundle/references/capture.md +++ b/packages/ghost/src/skill-bundle/references/capture.md @@ -181,7 +181,7 @@ action beats completeness... `materials`, a substantial fenced example, or a `## Skeleton`. You do not declare a separate type. -## What a body answers +## What a node body answers While drafting, ask three questions of every truth — *why* (the stance), *with what* (the materials), and *how it is assembled* (the patterns). These are @@ -204,7 +204,7 @@ genuinely a different truth. ## Node prose stances -Node prose is steering payload. A generic sentence in a body averages every +Node prose is steering payload. A generic sentence in a node body averages every future generation toward the median, so hold drafts to these stances before the human sees them. @@ -226,13 +226,13 @@ Everywhere else: fits a competitor's brand, it is retrieval-dead; rewrite it until it could belong to no one else. - **Cut unratified hedges.** "Generally," "where possible," and "consider" in a - body mean the human never picked a side. Get the ratification or cut the + node body mean the human never picked a side. Get the ratification or cut the sentence. - **Ban brand-deck filler.** "Elevate," "delight," "seamless," "best-in-class," "empower." When a brand doc supplies these words, they are testimony to distill, never prose to keep. - **Settle the altitude on purpose.** Every truth is either claimed universal - or given its condition in the prose. A body that does neither was never + or given its condition in the prose. A node body that does neither was never curated for altitude; ask the human which it is. ## Score drafts before curation @@ -244,7 +244,7 @@ dimension: | --- | --- | | Testimony | Can you quote the human words or artifact this node came from? | | Discrimination | Does the description fit only this brand? | -| Force | Does the body decide something, or merely describe something? | +| Force | Does the node body decide something, or merely describe something? | | Altitude | Is it universal on purpose, or given its condition? | | Residue | Is it free of starter-demo prose and brand-deck filler? | diff --git a/packages/ghost/test/cli.test.ts b/packages/ghost/test/cli.test.ts index f8cde5d7..ea313adb 100644 --- a/packages/ghost/test/cli.test.ts +++ b/packages/ghost/test/cli.test.ts @@ -210,40 +210,114 @@ describe("ghost CLI", () => { expect(result.stderr).toContain("--format json"); }); - it("initializes the default steering fingerprint package", async () => { - const init = await runCli(["init", "--format", "json"], dir); + const SKELETON_FILES = [ + "manifest.yml", + ".gitignore", + "glossary.md", + "index.md", + "anti-goal.median.md", + "grammar.hierarchy.md", + "grammar.rhythm.md", + "grammar.surfaces.md", + "grammar.motion.md", + "grammar.color-roles.md", + "grammar.conversation.md", + "signature.shape.md", + "signature.palette.md", + "signature.type.md", + "signature.temperature.md", + ]; - expect(init.code).toBe(0); - const initOutput = JSON.parse(init.stdout); - expect(Object.keys(initOutput).sort()).toEqual(["dir", "written"]); - expect(initOutput.written).toContain("manifest.yml"); - expect(initOutput.written).toContain("glossary.md"); - expect(initOutput.written).toContain("index.md"); - expect(initOutput.written).toContain("principle.stance.md"); - expect(initOutput.written).toContain("principle.composition.md"); - expect(initOutput.written).toContain( - "anti-goal.generic-generated-output.md", - ); - expect(initOutput.written).toContain("pattern.status-with-next-step.md"); - expect(initOutput.written).toContain("exemplar.annotated-reference.md"); - expect(initOutput.written).toContain("asset.materials.md"); - expect(initOutput.written).toContain("decision.tradeoff.md"); + async function expectSkeletonPackage(written: string[]) { + // Exact file inventory: no anti-goal.tells, no register.*, no materials/. + expect([...written].sort()).toEqual([...SKELETON_FILES].sort()); + expect(written).not.toContain("anti-goal.tells.md"); + expect(written.some((f: string) => f.startsWith("register."))).toBe(false); + expect(written.some((f: string) => f.startsWith("materials/"))).toBe(false); // Core init is fingerprint-only: checks are opt-in via --with / checks init. - expect(initOutput.written).not.toContain("checks/example.md.example"); + expect(written).not.toContain("checks/example.md.example"); + await expect( readFile(join(dir, ".ghost", "manifest.yml"), "utf-8"), ).resolves.toContain("schema: ghost.fingerprint-package/v1"); - const index = await readFile(join(dir, ".ghost", "index.md"), "utf-8"); - expect(index).toContain( - "Everything in this starter package is demo content", - ); - expect(index).toContain( - "replace the claims, paths, examples, and decisions", + // The scaffolded package validates with zero errors AND zero warnings. + const validate = await runCli(["validate", "--format", "json"], dir); + expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.errors).toBe(0); + expect(report.warnings).toBe(0); + + // The gather menu carries every node with its kind. + const gather = await runCli(["gather", "--format", "json"], dir); + expect(gather.code).toBe(0); + const menu = JSON.parse(gather.stdout); + const byId = new Map( + menu.nodes.map((node: { id: string; kind?: string }) => [ + node.id, + node.kind, + ]), ); + expect(byId.get("index")).toBeUndefined; + expect(byId.has("index")).toBe(true); + expect(byId.get("anti-goal.median")).toBe("anti-goal"); + for (const slug of [ + "hierarchy", + "rhythm", + "surfaces", + "motion", + "color-roles", + "conversation", + ]) { + expect(byId.get(`grammar.${slug}`)).toBe("grammar"); + } + for (const slug of ["shape", "palette", "type", "temperature"]) { + expect(byId.get(`signature.${slug}`)).toBe("signature"); + } - const validate = await runCli(["validate"], dir); - expect(validate.code).toBe(0); + // The median floor survives intact: prune header + rule anchors. + const median = await runCli(["pull", "anti-goal.median"], dir); + expect(median.code).toBe(0); + expect(median.stdout).toContain( + "This is the model's median, not your brand.", + ); + expect(median.stdout).toContain("### Side-stripe"); + + // The dials ship unanswered and forbid freehanding. + const shape = await runCli(["pull", "signature.shape"], dir); + expect(shape.code).toBe(0); + expect(shape.stdout).toContain("This dial is unanswered"); + expect(shape.stdout).toContain("freehand"); + + // No Vessel strings anywhere in the scaffolded package. + const forbidden = [ + "Vessel", + "HK Grotesk", + "999px", + "Morrow", + "amber", + "periwinkle", + "clay", + "orchid", + "sage", + ]; + for (const file of written) { + const content = await readFile(join(dir, ".ghost", file), "utf-8"); + for (const needle of forbidden) { + expect(content, `${file} must not contain "${needle}"`).not.toMatch( + new RegExp(`\\b${needle}\\b`, "i"), + ); + } + } + } + + it("initializes the default skeleton fingerprint package", async () => { + const init = await runCli(["init", "--format", "json"], dir); + + expect(init.code).toBe(0); + const initOutput = JSON.parse(init.stdout); + expect(Object.keys(initOutput).sort()).toEqual(["dir", "written"]); + await expectSkeletonPackage(initOutput.written); }); it("initializes the minimal fingerprint package", async () => { @@ -257,26 +331,34 @@ describe("ghost CLI", () => { expect(initOutput.written).toContain("manifest.yml"); expect(initOutput.written).toContain("glossary.md"); expect(initOutput.written).toContain("index.md"); + expect(initOutput.written).toContain("anti-goal.median.md"); expect(initOutput.written).not.toContain("principle.stance.md"); expect(initOutput.written).not.toContain("decision.tradeoff.md"); - const validate = await runCli(["validate"], dir); + const validate = await runCli(["validate", "--format", "json"], dir); expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.errors).toBe(0); + expect(report.warnings).toBe(0); }); - it("keeps default as an alias for the steering template", async () => { - const init = await runCli( - ["init", "--template", "default", "--format", "json"], - dir, - ); + it("keeps default and steering as aliases for the skeleton template", async () => { + for (const name of ["default", "steering"]) { + await rm(join(dir, ".ghost"), { recursive: true, force: true }); + const init = await runCli( + ["init", "--template", name, "--format", "json"], + dir, + ); - expect(init.code).toBe(0); - const initOutput = JSON.parse(init.stdout); - expect(initOutput.written).toContain("principle.stance.md"); - expect(initOutput.written).toContain("decision.tradeoff.md"); + expect(init.code).toBe(0); + const initOutput = JSON.parse(init.stdout); + expect([...initOutput.written].sort()).toEqual( + [...SKELETON_FILES].sort(), + ); - const validate = await runCli(["validate"], dir); - expect(validate.code).toBe(0); + const validate = await runCli(["validate"], dir); + expect(validate.code).toBe(0); + } }); it("initializes the composition starter template", async () => { @@ -292,10 +374,14 @@ describe("ghost CLI", () => { expect(initOutput.written).toContain("index.md"); expect(initOutput.written).toContain("principle.composition.md"); expect(initOutput.written).toContain("pattern.status-with-next-step.md"); + expect(initOutput.written).toContain("anti-goal.median.md"); // The scaffolded package is valid as written. - const validate = await runCli(["validate"], dir); + const validate = await runCli(["validate", "--format", "json"], dir); expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.errors).toBe(0); + expect(report.warnings).toBe(0); // The ladder nodes surface in the gather menu with their kinds. const gather = await runCli(["gather", "--format", "json"], dir); @@ -317,55 +403,15 @@ describe("ghost CLI", () => { expect(pull.stdout).toContain("principle.composition"); }); - it("initializes the steering starter template", async () => { + it("initializes the skeleton starter template by name", async () => { const init = await runCli( - ["init", "--template", "steering", "--format", "json"], + ["init", "--template", "skeleton", "--format", "json"], dir, ); expect(init.code).toBe(0); const initOutput = JSON.parse(init.stdout); - expect(initOutput.written).toContain("manifest.yml"); - expect(initOutput.written).toContain("glossary.md"); - expect(initOutput.written).toContain("index.md"); - expect(initOutput.written).toContain("principle.stance.md"); - expect(initOutput.written).toContain("principle.composition.md"); - expect(initOutput.written).toContain( - "anti-goal.generic-generated-output.md", - ); - expect(initOutput.written).toContain("pattern.status-with-next-step.md"); - expect(initOutput.written).toContain("exemplar.annotated-reference.md"); - expect(initOutput.written).toContain("asset.materials.md"); - expect(initOutput.written).toContain("decision.tradeoff.md"); - expect(initOutput.written).not.toContain("concept.scoped-direction.md"); - - const validate = await runCli(["validate"], dir); - expect(validate.code).toBe(0); - - const gather = await runCli(["gather", "--format", "json"], dir); - expect(gather.code).toBe(0); - const menu = JSON.parse(gather.stdout); - const ids = menu.nodes.map((node: { id: string }) => node.id); - expect(ids).toContain("principle.stance"); - expect(ids).toContain("principle.composition"); - expect(ids).toContain("anti-goal.generic-generated-output"); - expect(ids).toContain("pattern.status-with-next-step"); - expect(ids).toContain("exemplar.annotated-reference"); - expect(ids).toContain("asset.materials"); - expect(ids).toContain("decision.tradeoff"); - - const antiGoal = menu.nodes.find( - (node: { id: string }) => - node.id === "anti-goal.generic-generated-output", - ); - expect(antiGoal.kind).toBe("anti-goal"); - const decision = menu.nodes.find( - (node: { id: string }) => node.id === "decision.tradeoff", - ); - expect(decision.kind).toBe("decision"); - expect(menu.kinds.map((kind: { name: string }) => kind.name)).toContain( - "concept", - ); + await expectSkeletonPackage(initOutput.written); }); it("rejects an unknown init template with a usage error", async () => { @@ -379,7 +425,84 @@ describe("ghost CLI", () => { expect(result.stderr).toContain("Unknown init template 'nope'"); expect(result.stderr).toContain("minimal"); expect(result.stderr).toContain("composition"); - expect(result.stderr).toContain("steering"); + expect(result.stderr).toContain("skeleton"); + }); + + it("installs the vessel-light body: full corpus, materials, checks", async () => { + const init = await runCli( + ["init", "--body", "vessel-light", "--format", "json"], + dir, + ); + expect(init.code).toBe(0); + const { written } = JSON.parse(init.stdout) as { written: string[] }; + + // The body is the inhabited package: corpus + tells + registers + + // materials tree + its own checks. No .events tape. + expect(written).toContain("manifest.yml"); + expect(written).toContain("anti-goal.median.md"); + expect(written).toContain("anti-goal.tells.md"); + expect(written).toContain("register.email.md"); + expect(written).toContain("signature.shape.md"); + expect(written).toContain("materials/tokens.css"); + expect(written).toContain("materials/fonts/HKGrotesk-Regular.woff2"); + expect(written).toContain("materials/ref/composition.form.html"); + expect(written).toContain("checks/median-tells.md"); + expect(written).toContain("checks/values.md"); + expect(written.some((p) => p.includes(".events"))).toBe(false); + + // Manifest id stays vessel-light: renaming it is step one of adapting + // the starter — an explicit human act, never pre-executed by init. + const manifest = await readFile( + join(dir, ".ghost", "manifest.yml"), + "utf-8", + ); + expect(manifest).toContain("id: vessel-light"); + + // Fonts survive the packed payload byte-identically. + const [installed, source] = await Promise.all([ + readFile( + join(dir, ".ghost", "materials", "fonts", "HKGrotesk-Regular.woff2"), + ), + readFile( + new URL( + "../../vessel-light/.ghost/materials/fonts/HKGrotesk-Regular.woff2", + import.meta.url, + ), + ), + ]); + expect(installed.equals(source)).toBe(true); + + // The installed body validates clean, checks included. + const validate = await runCli(["validate", "--format", "json"], dir); + expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.errors).toBe(0); + expect(report.warnings).toBe(0); + }); + + it("rejects unknown bodies and contradictory body flags", async () => { + const unknown = await runCli(["init", "--body", "nope"], dir, { + allowNoExit: true, + }); + expect(unknown.code).toBe(2); + expect(unknown.stderr).toContain("Unknown init body 'nope'"); + expect(unknown.stderr).toContain("vessel-light"); + + const both = await runCli( + ["init", "--body", "vessel-light", "--template", "minimal"], + dir, + { allowNoExit: true }, + ); + expect(both.code).toBe(2); + expect(both.stderr).toContain("mutually exclusive"); + + const withChecks = await runCli( + ["init", "--body", "vessel-light", "--with", "checks"], + dir, + { allowNoExit: true }, + ); + expect(withChecks.code).toBe(2); + expect(withChecks.stderr).toContain("already includes its own checks/"); }); it("uses GHOST_PACKAGE_DIR as the default fingerprint package directory for init", async () => { @@ -501,7 +624,7 @@ describe("ghost CLI", () => { expect(forced.code).toBe(0); await expect( readFile(join(dir, ".ghost", "index.md"), "utf-8"), - ).resolves.toContain("Everything in this starter package is demo content"); + ).resolves.toContain("skeleton fingerprint"); }); it("does not guess arbitrary YAML files are validate.yml", async () => { @@ -560,7 +683,7 @@ describe("ghost CLI", () => { it("rejects the removed --reference init flag", async () => { await expect( - runCli(["init", "--reference", "packages/vessel/.ghost"], dir), + runCli(["init", "--reference", "packages/vessel-react/.ghost"], dir), ).rejects.toThrow("Unknown option `--reference`"); }); @@ -586,15 +709,17 @@ describe("ghost CLI", () => { const gather = await runCli(["gather", "--format", "json"], dir); expect(gather.code).toBe(0); const menu = JSON.parse(gather.stdout); - const principle = menu.kinds.find( - (k: { name: string }) => k.name === "principle", + const grammar = menu.kinds.find( + (k: { name: string }) => k.name === "grammar", ); - expect(principle.purpose).toContain("Durable stance"); + expect(grammar.purpose).toContain("decision logic"); // Markdown renders the same legend above the node list. const markdown = await runCli(["gather"], dir); expect(markdown.stdout).toContain("Kinds:"); - expect(markdown.stdout).toContain("- **principle** — Durable stance"); + expect(markdown.stdout).toContain( + "- **grammar** — The brand's decision logic", + ); // A missing glossary degrades to no legend, not an error. await rm(join(dir, ".ghost", "glossary.md")); @@ -791,13 +916,13 @@ a deliberate provocation past the fingerprint — surfaced only on request const gather = await runCli(["gather", "--format", "json"], dir); expect(gather.code).toBe(0); expect(JSON.parse(gather.stdout).coverage).toEqual({ - nodes: 4, + nodes: 5, concrete: 1, - guards: 1, + guards: 2, }); const markdown = await runCli(["gather"], dir); expect(markdown.stdout).toContain( - "4 nodes · 1 carry concrete material · 1 guards", + "5 nodes · 1 carry concrete material · 2 guards", ); const steering = await runCli( @@ -1486,6 +1611,54 @@ a deliberate provocation past the fingerprint — surfaced only on request }); }); + it("review resolves package-relative locators when the package sits below the repo root", async () => { + // Regression: exact-path `materials/…` locators were matched as raw text + // against repo-relative diff paths, so a package below the repo root + // (e.g. packages/vessel-light/.ghost) never matched them — its value + // checks were silently dropped from the packet. + const packageDir = join("nested", "app", ".ghost"); + await runCli(["init", "--package", packageDir], dir); + await mkdir(join(dir, packageDir, "materials"), { recursive: true }); + await writeFile( + join(dir, packageDir, "materials", "tokens.css"), + ":root{}\n", + ); + await writeFile( + join(dir, packageDir, "asset.tokens.md"), + "---\ndescription: Tokens.\nmaterials:\n - materials/tokens.css\n---\n\nTokens prose.\n", + ); + await mkdir(join(dir, packageDir, "checks"), { recursive: true }); + await writeFile( + join(dir, packageDir, "checks", "token-discipline.md"), + "---\nname: token-discipline\ndescription: Tokens hold.\nseverity: high\nreferences:\n - asset.tokens\n---\n\nGrade token discipline.\n", + ); + const touched = `${packageDir.replaceAll("\\", "/")}/materials/tokens.css`; + const diff = [ + `diff --git a/${touched} b/${touched}`, + `--- a/${touched}`, + `+++ b/${touched}`, + "@@ -1 +1 @@", + "-old", + "+new", + ].join("\n"); + + const result = await runCli( + ["review", "--package", packageDir, "--diff=-", "--format", "json"], + dir, + { stdin: diff }, + ); + + expect(result.code).toBe(0); + const packet = JSON.parse(result.stdout); + expect(packet.materialNodes.map((n: { id: string }) => n.id)).toContain( + "asset.tokens", + ); + const check = packet.checks.find( + (c: { id: string }) => c.id === "token-discipline", + ); + expect(check).toMatchObject({ offered: "matched" }); + }); + it("review runs check probes as evidence and supports --no-probes", async () => { await runCli(["init", "--with", "checks"], dir); await writeFile( @@ -1704,25 +1877,92 @@ a deliberate provocation past the fingerprint — surfaced only on request expect(pull.stdout).toContain(":root{}"); }); - it("checks init scaffolds .ghost/checks/ with an example", async () => { + it("checks init scaffolds .ghost/checks/ with median tells and an example", async () => { await runCli(["init"], dir); const add = await runCli(["checks", "init", "--format", "json"], dir); expect(add.code).toBe(0); const added = JSON.parse(add.stdout); - expect(added.written).toEqual(["example.md.example"]); + expect(added.written).toEqual(["median-tells.md", "example.md.example"]); + expect(added.skipped).toEqual([]); await expect( readFile(join(dir, ".ghost", "checks", "example.md.example"), "utf-8"), ).resolves.toContain("references:"); + // The live median check pairs with the skeleton's anti-goal.median node. + const median = await readFile( + join(dir, ".ghost", "checks", "median-tells.md"), + "utf-8", + ); + expect(median).toContain("anti-goal.median"); + expect(median).toContain("anti-goal.median > Hover-lift"); + expect(median).toContain("prefers-reduced-motion"); + expect(median).toContain( + "`ghost validate` warns; delete the flag and its reference together.", + ); + expect(median).not.toContain("Vessel"); + // Running init twice is a usage error. const again = await runCli(["checks", "init"], dir); expect(again.code).toBe(2); expect(again.stderr).toContain("already exists"); - // The scaffold validates cleanly (example.md.example is inert). - const validate = await runCli(["validate"], dir); + // The scaffold validates cleanly on the default skeleton: median-tells + // references resolve against anti-goal.median and the grammar nodes. + const validate = await runCli(["validate", "--format", "json"], dir); expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + const unresolved = report.issues.filter( + (f: { rule: string }) => f.rule === "check-reference-unresolved", + ); + expect(unresolved).toEqual([]); + }); + + it("checks init skips median tells when the median node is absent", async () => { + await runCli(["init", "--template", "minimal"], dir); + await rm(join(dir, ".ghost", "anti-goal.median.md")); + + const add = await runCli(["checks", "init"], dir); + expect(add.code).toBe(0); + expect(add.stdout).toContain( + "skipped median-tells.md (no anti-goal.median node)", + ); + + await expect( + readFile(join(dir, ".ghost", "checks", "median-tells.md"), "utf-8"), + ).rejects.toThrow(); + + const validate = await runCli(["validate", "--format", "json"], dir); + expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.errors).toBe(0); + expect(report.warnings).toBe(0); + }); + + it("validate warns when a pruned median heading orphans its paired check", async () => { + await runCli(["init"], dir); + await runCli(["checks", "init"], dir); + const path = join(dir, ".ghost", "anti-goal.median.md"); + const median = await readFile(path, "utf-8"); + await writeFile( + path, + median.replace(/### Side-stripe\n[\s\S]*?(?=\n### Cream surface)/, ""), + ); + + const validate = await runCli(["validate", "--format", "json"], dir); + expect(validate.code).toBe(0); + const report = JSON.parse(validate.stdout); + expect(report.warnings).toBe(1); + expect(report.issues).toEqual([ + expect.objectContaining({ + severity: "warning", + rule: "check-reference-heading-missing", + message: expect.stringContaining("anti-goal.median > Side-stripe"), + }), + ]); + expect(report.issues[0].message).toContain( + "if you pruned this rule from the node, delete its paired flag in the check too", + ); }); it("checks rejects unknown actions", async () => { diff --git a/packages/steering-eval/README.md b/packages/steering-eval/README.md new file mode 100644 index 00000000..7593b829 --- /dev/null +++ b/packages/steering-eval/README.md @@ -0,0 +1,131 @@ +# steering-eval + +A before/after evaluation harness for Ghost fingerprints. It measures what +handing an agent a `.ghost` fingerprint buys — output quality, run-to-run +consistency, retrieval precision — as a ratio of the context tokens spent. + +It produces one artifact: a self-contained `report.html` with screenshot +grids, variance bands, gather metrics, and a headline quality-vs-context +chart. Every number is deterministic — grep-able tells, selection-tape +diffs, CSS extraction. No LLM judges. Judgment stays with the human reading +the report. + +## The arms + +| Arm | Context the generating agent sees | Claim it tests | +| --- | --- | --- | +| `naked` | ballast + ask | baseline: median model output, no brand | +| `dump` | full fingerprint prose up front + ballast + ask | the naive "paste the brand guide in the system prompt" | +| `gather` | ballast + `ghost gather` menu + agent-selected `ghost pull`s near the ask | Ghost's actual loop | +| `dump-growth` | dump of this fingerprint **plus** extra corpora + ballast + ask | how dumping degrades as the corpus grows, while the gather menu grows one line per node | + +Fresh context per run. The `naked` arm must never see the fingerprint. + +## Setup + +You need: Node 20+, the `ghost` CLI on PATH (or set `ghostBin` in config), +a `.ghost/` fingerprint package, and an agent that drives the loop (Claude +Code, goose, Cursor — anything that can read a prompt file and write HTML). + +```bash +npx steering-eval init # writes eval.config.json + asks.md templates +``` + +Fill in `eval.config.json`: + +```json +{ + "package": "./.ghost", + "asks": "./asks.md", + "ballast": "./ballast.md", + "runsPerCell": 5, + "arms": { + "naked": true, + "dump": true, + "gather": true, + "dump-growth": { "extraPackages": ["../other/.ghost"], "asks": [1] } + }, + "tells": null, + "out": "./out", + "ghostBin": "ghost" +} +``` + +- `package` — the fingerprint under test. +- `asks` — a markdown file of numbered asks (see format below). +- `ballast` — fixed, realistic-shaped irrelevant context, identical across + arms. It exists to create window pressure. Never edit it between runs. +- `tells` — path to a custom tells JSON, or `null` for the built-in + model-median tells (measured defaults of unsteered generation). +- `dump-growth.extraPackages` — real corpora only. Label the point honestly; + do not pad synthetically. + +## Asks format + +```markdown +## Ask 1 — billing settings page + +Build a billing settings page for a SaaS product. ... + +expect: foundation.tokens, primitive.control, primitive.stack +poison: pattern.email, pattern.editorial +``` + +- `expect:` — node ids a correct selection should include. Keep the set + minimal and defensible: if reasonable people could argue about a node, + leave it out. Contested nodes score as neutral. +- `poison:` — nodes whose selection is a retrieval failure (wrong register). + "Pulled the email rules while building a settings page" is the headline + retrieval metric. + +## The loop + +The harness assembles prompts and keeps books. **Your agent generates.** + +Per cell (arm × ask), for k = 1..runsPerCell: + +```bash +steering-eval prompt --run # writes out//ask-/run-.prompt.md +# → hand the prompt file to a FRESH agent context; it writes run-.html +# (gather arm: the agent runs `ghost pull` itself mid-task — that IS the test) +steering-eval finish # slices the selection tape, records context sizes +``` + +Gather-arm runs are strictly serialized — the harness hard-fails if a run +is already open, because the selection tape is append-only and concurrent +runs corrupt attribution. + +Then, zero LLM calls: + +```bash +steering-eval shoot # screenshots every out/**/*.html via agent-browser (idempotent) +steering-eval score # out/metrics.json — distributions, consistency, retrieval +steering-eval report # out/report.html — self-contained, rebuildable from out/ alone +``` + +## What the report shows + +- **Headline chart** — X: brand-context tokens (estimated, words x 1.33, + labeled as such), Y: inverted median-tell score. One point per arm with a + min-max variance band. The dump-growth point shows the scaling story. +- **Per-ask screenshot grids** — arms x runs. Variance you can see. +- **Range bands** — min / median / max tell score per cell. The claim is + that steering collapses the spread, not just the mean. +- **Style consistency** — do the k runs agree on accent hue, radius, font + stack? Fraction agreeing on the modal value, per dimension. +- **Gather metrics** (gather arm only, from the selection tape) — poison + pulls (the legible number), precision/recall (secondary), selection + stability across runs (mean pairwise Jaccard), pulled words per run. +- **Reproduction footer** — the exact commands to rebuild the report from + `out/`, and the note that no LLM judged anything. + +## Honest-claims checklist + +- Report distributions, never single wins. +- The token axis is an estimate; the method note says so. +- If your ballast is small, claim "mid-conversation pressure," not "heavy + context." Dilution claims need real window pressure (~80K+ tokens). +- Custom tells must be measured, not aesthetic opinions. The built-in set + comes from 300 unsteered generations across three models. +- Commit `out/` (prompts, meta, metrics, report) so skeptics can rerun any + arm and audit any number. diff --git a/packages/steering-eval/cli.mjs b/packages/steering-eval/cli.mjs new file mode 100755 index 00000000..7f9fa2ab --- /dev/null +++ b/packages/steering-eval/cli.mjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { assemblePrompt, writePrompt } from "./lib/arms.mjs"; +import { loadConfig } from "./lib/config.mjs"; +import { scoreAll } from "./lib/score.mjs"; +import { shootAll } from "./lib/shoot.mjs"; +import { closeRun, openRun } from "./lib/tape.mjs"; + +const args = process.argv.slice(2); +const command = args[0]; + +try { + if (!command || command === "--help" || command === "-h") usage(); + else if (command === "init") init(); + else if (command === "prompt") prompt(args.slice(1)); + else if (command === "finish") finish(args.slice(1)); + else if (command === "shoot") shootAll(loadConfig(process.cwd())); + else if (command === "score") scoreAll(loadConfig(process.cwd())); + else if (command === "report") await report(); + else { + console.error(`unknown command: ${command}`); + usage(1); + } +} catch (error) { + console.error(error.message); + process.exit(1); +} + +function usage(exitCode = 0) { + console.log(`usage: steering-eval [args] + +Commands: + init + prompt --run + finish + shoot + score + report + --help`); + process.exit(exitCode); +} + +function init() { + const files = { + "eval.config.json": `${JSON.stringify(starterConfig(), null, 2)}\n`, + "asks.md": `## Ask 1 — billing settings page\n\nBuild a billing settings page for a SaaS product.\n\nexpect: foundation.tokens, primitive.control\npoison: pattern.email\ndiscount:\n`, + "ballast.md": `This is fixed ballast context. Replace it with realistic irrelevant project context before running an evaluation.\n`, + }; + for (const file of Object.keys(files)) { + if (existsSync(file)) throw new Error(`refusing to overwrite ${file}`); + } + for (const [file, content] of Object.entries(files)) + writeFileSync(file, content, "utf8"); + console.log("wrote eval.config.json, asks.md, ballast.md"); +} + +function prompt(argv) { + const [arm, askN] = argv; + const runK = readRun(argv); + if (!arm || !askN || !runK) usage(1); + const config = loadConfig(process.cwd()); + openRun(config, arm, Number(askN), Number(runK)); + const assembled = assemblePrompt(config, arm, Number(askN), Number(runK)); + const paths = writePrompt(config, arm, Number(askN), Number(runK), assembled); + console.log(`prompt: ${paths.promptPath}`); + console.log(`output: ${paths.outputPath}`); +} + +function finish(argv) { + const [arm, askN, runK] = argv; + if (!arm || !askN || !runK) usage(1); + const config = loadConfig(process.cwd()); + const tape = closeRun(config); + const dir = join(config.out, arm, `ask-${askN}`); + const htmlPath = join(dir, `run-${runK}.html`); + if (!existsSync(htmlPath)) + console.warn(`warning: expected output HTML not found: ${htmlPath}`); + const promptPath = join(dir, `run-${runK}.prompt.md`); + const inventoryPath = promptPath.replace(/\.prompt\.md$/u, ".inventory.json"); + const inventory = existsSync(inventoryPath) + ? JSON.parse(readFileSync(inventoryPath, "utf8")) + : existsSync(promptPath) + ? inventoryFromPrompt(readFileSync(promptPath, "utf8")) + : {}; + const meta = { arm, askN: Number(askN), runK: Number(runK), inventory, tape }; + writeFileSync( + join(dir, `run-${runK}.meta.json`), + JSON.stringify(meta, null, 2), + "utf8", + ); + console.log(`meta: ${join(dir, `run-${runK}.meta.json`)}`); +} + +async function report() { + const config = loadConfig(process.cwd()); + const metricsJsonPath = join(config.out, "metrics.json"); + try { + const mod = await import("./lib/report.mjs"); + await mod.renderReport(metricsJsonPath, config.out); + console.log(`report: ${join(config.out, "report.html")}`); + } catch (error) { + if (error.code === "ERR_MODULE_NOT_FOUND") { + console.error("report module not yet available"); + process.exit(1); + } + throw error; + } +} + +function readRun(argv) { + const index = argv.indexOf("--run"); + return index >= 0 ? argv[index + 1] : null; +} +function starterConfig() { + return { + package: "./.ghost", + asks: "./asks.md", + ballast: "./ballast.md", + runsPerCell: 5, + arms: { + naked: true, + dump: true, + gather: true, + "dump-growth": { extraPackages: ["../other/.ghost"], asks: [1] }, + }, + tells: null, + out: "./out", + ghostBin: "ghost", + }; +} +function inventoryFromPrompt(prompt) { + const total = (prompt.match(/\S+/gu) ?? []).length; + return { total, tokensEstimate: Math.round(total * 1.33) }; +} diff --git a/packages/steering-eval/default-tells.json b/packages/steering-eval/default-tells.json new file mode 100644 index 00000000..13213370 --- /dev/null +++ b/packages/steering-eval/default-tells.json @@ -0,0 +1,97 @@ +{ + "source": "antimedian (300 unsteered generations, gemini/gpt/opus, 2026-07)", + "note": "counts are antimedian pattern occurrences; weight reflects distinctiveness-destroying power. Sanctioned-by-vessel median patterns (pills, clamp headings, max-width containers, email tables) are deliberately absent.", + "tells": [ + { + "id": "hover-lift-family", + "antimedianCount": 341, + "kind": "regex", + "pattern": ":hover[^}]*transform:[^};]*translateY\\(-", + "weight": 3 + }, + { + "id": "indigo-blue-default-accent", + "antimedianCount": 60, + "kind": "regex", + "pattern": "#(4f46e5|6366f1|4338ca|2563eb|3b82f6|1d4ed8|4a90e2|4361ee)\\b", + "weight": 3 + }, + { + "id": "unprompted-dark-theme", + "antimedianCount": 271, + "kind": "regex", + "pattern": "(?:(?:^|[}\\s,])(?:body|html|:root)\\s*(?:,[^{}]*)?\\{[^}]*background(?:-color)?:\\s*(?:#(?:[0-2][0-9a-fA-F]){3}\\b|#[0-2]{3}\\b|black\\b|rgba?\\(\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d)|--(?:bg|background|surface)[\\w-]*:\\s*(?:#(?:[0-2][0-9a-fA-F]){3}\\b|#[0-2]{3}\\b|black\\b|rgba?\\(\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d))", + "weight": 2 + }, + { + "id": "gradient-background", + "antimedianCount": 63, + "kind": "regex", + "pattern": "background[^;]*:(linear|radial)-gradient|background-image:\\s*(linear|radial)-gradient", + "weight": 3 + }, + { + "id": "glassmorphism", + "antimedianCount": 23, + "kind": "regex", + "pattern": "backdrop-filter:\\s*blur", + "weight": 2 + }, + { + "id": "chat-bubble-avatar", + "antimedianCount": 57, + "kind": "regex", + "pattern": "(avatar[^}]*border-radius:\\s*50%|initials)", + "weight": 1 + }, + { + "id": "emoji-icons", + "antimedianCount": 26, + "kind": "regex", + "pattern": "[\\u{1F300}-\\u{1FAFF}\\u{2600}-\\u{27BF}]", + "weight": 1 + }, + { + "id": "stock-pricing-copy", + "antimedianCount": 14, + "kind": "text", + "pattern": "Simple, transparent pricing", + "weight": 2 + }, + { + "id": "welcome-back-copy", + "antimedianCount": 10, + "kind": "text", + "pattern": "Welcome back", + "weight": 1 + }, + { + "id": "sticky-top-header", + "antimedianCount": 239, + "kind": "regex", + "pattern": "(header|nav)[^}]*position:\\s*(sticky|fixed)[^}]*top:\\s*0", + "weight": 1 + }, + { + "id": "viewport-centered-card", + "antimedianCount": 152, + "kind": "regex", + "pattern": "(body|\\.container)[^}]*display:\\s*flex[^}]*(align-items:\\s*center[^}]*justify-content:\\s*center|justify-content:\\s*center[^}]*align-items:\\s*center)[^}]*(min-)?height:\\s*100(vh|dvh)", + "weight": 1 + }, + { + "id": "inter-font-default", + "antimedianCount": 44, + "kind": "regex", + "pattern": "font-family:[^;]*\\bInter\\b", + "weight": 1 + }, + { + "id": "segoe-font-default", + "antimedianCount": 104, + "kind": "regex", + "pattern": "font-family:[^;]*Segoe UI", + "weight": 1 + } + ] +} diff --git a/packages/steering-eval/fixtures/metrics.sample.json b/packages/steering-eval/fixtures/metrics.sample.json new file mode 100644 index 00000000..8f610751 --- /dev/null +++ b/packages/steering-eval/fixtures/metrics.sample.json @@ -0,0 +1,210 @@ +{ + "generatedAt": "2026-07-08T09:00:00.000Z", + "config": { "runsPerCell": 5, "package": "../.ghost" }, + "cells": [ + { + "arm": "naked", + "ask": 1, + "askTitle": "billing settings page", + "scores": { + "min": 12, + "median": 17, + "max": 21, + "mean": 16.6, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.2, "radius": 0.4, "fontStack": 0.4 }, + "context": { + "tokensEstimate": 80150, + "segments": { "ballast": 80000, "brand": 0, "menu": 0, "ask": 150 } + } + }, + { + "arm": "dump", + "ask": 1, + "askTitle": "billing settings page", + "scores": { + "min": 6, + "median": 10, + "max": 15, + "mean": 10.2, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.6, "radius": 0.6, "fontStack": 0.8 }, + "context": { + "tokensEstimate": 85150, + "segments": { "ballast": 80000, "brand": 5000, "menu": 0, "ask": 150 } + } + }, + { + "arm": "gather", + "ask": 1, + "askTitle": "billing settings page", + "scores": { + "min": 3, + "median": 5, + "max": 8, + "mean": 5.2, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 1.0, "radius": 0.8, "fontStack": 1.0 }, + "context": { + "tokensEstimate": 87350, + "segments": { "ballast": 80000, "brand": 0, "menu": 700, "ask": 150 } + }, + "retrieval": { + "precision": 0.86, + "recall": 0.9, + "poisonPulls": 0, + "poisonRuns": 0, + "stability": 0.88, + "pulledWordsMean": 4900 + } + }, + { + "arm": "dump-growth", + "ask": 1, + "askTitle": "billing settings page", + "scores": { + "min": 8, + "median": 12, + "max": 18, + "mean": 12.4, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.4, "radius": 0.4, "fontStack": 0.6 }, + "context": { + "tokensEstimate": 91150, + "segments": { "ballast": 80000, "brand": 11000, "menu": 0, "ask": 150 } + } + }, + { + "arm": "naked", + "ask": 2, + "askTitle": "pricing landing page", + "scores": { + "min": 11, + "median": 16, + "max": 22, + "mean": 16.2, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.2, "radius": 0.2, "fontStack": 0.6 }, + "context": { + "tokensEstimate": 80180, + "segments": { "ballast": 80000, "brand": 0, "menu": 0, "ask": 180 } + } + }, + { + "arm": "dump", + "ask": 2, + "askTitle": "pricing landing page", + "scores": { + "min": 5, + "median": 9, + "max": 14, + "mean": 9.4, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.6, "radius": 0.8, "fontStack": 0.8 }, + "context": { + "tokensEstimate": 85180, + "segments": { "ballast": 80000, "brand": 5000, "menu": 0, "ask": 180 } + } + }, + { + "arm": "gather", + "ask": 2, + "askTitle": "pricing landing page", + "scores": { + "min": 2, + "median": 4, + "max": 7, + "mean": 4.4, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 1.0, "radius": 1.0, "fontStack": 1.0 }, + "context": { + "tokensEstimate": 88080, + "segments": { "ballast": 80000, "brand": 0, "menu": 700, "ask": 180 } + }, + "retrieval": { + "precision": 0.82, + "recall": 0.85, + "poisonPulls": 0, + "poisonRuns": 0, + "stability": 0.84, + "pulledWordsMean": 5400 + } + }, + { + "arm": "naked", + "ask": 3, + "askTitle": "receipt email", + "scores": { + "min": 10, + "median": 15, + "max": 20, + "mean": 15.0, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.4, "radius": 0.2, "fontStack": 0.4 }, + "context": { + "tokensEstimate": 80120, + "segments": { "ballast": 80000, "brand": 0, "menu": 0, "ask": 120 } + } + }, + { + "arm": "dump", + "ask": 3, + "askTitle": "receipt email", + "scores": { + "min": 6, + "median": 11, + "max": 16, + "mean": 10.8, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.6, "radius": 0.6, "fontStack": 0.6 }, + "context": { + "tokensEstimate": 85120, + "segments": { "ballast": 80000, "brand": 5000, "menu": 0, "ask": 120 } + } + }, + { + "arm": "gather", + "ask": 3, + "askTitle": "receipt email", + "scores": { + "min": 3, + "median": 6, + "max": 9, + "mean": 6.0, + "n": 5, + "max_possible": 22 + }, + "consistency": { "accentHue": 0.8, "radius": 0.8, "fontStack": 1.0 }, + "context": { + "tokensEstimate": 87720, + "segments": { "ballast": 80000, "brand": 0, "menu": 700, "ask": 120 } + }, + "retrieval": { + "precision": 0.74, + "recall": 0.8, + "poisonPulls": 1, + "poisonRuns": 1, + "stability": 0.79, + "pulledWordsMean": 4700 + } + } + ] +} diff --git a/packages/steering-eval/lib/arms.mjs b/packages/steering-eval/lib/arms.mjs new file mode 100644 index 00000000..f2d70dcd --- /dev/null +++ b/packages/steering-eval/lib/arms.mjs @@ -0,0 +1,136 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; +import { parseAsks } from "./asks.mjs"; + +export function assemblePrompt(config, arm, askN, runK) { + const ask = findAsk(config, askN); + const ballast = readFileSync(config.ballast, "utf8").trim(); + const outputPath = htmlPath(config, arm, askN, runK); + const segments = []; + let brand = ""; + let menu = ""; + + if (arm === "dump" || arm === "dump-growth") { + brand = dumpPackage(config.package); + if (arm === "dump-growth") { + for (const extra of config.arms["dump-growth"]?.extraPackages ?? []) { + brand += `\n\n${dumpPackage(extra)}`; + } + } + segments.push(section("Brand fingerprint", brand)); + } else if (arm === "gather") { + const [ghostCmd, ...ghostPre] = config.ghostBin.split(/\s+/u); + menu = execFileSync( + ghostCmd, + [ + ...ghostPre, + ...(config.ghostArgs ?? []), + "gather", + firstLine(ask.body), + "--package", + config.package, + ], + { encoding: "utf8" }, + ); + segments.push(section("Ghost gather menu", menu.trim())); + segments.push( + section("Selection instructions", gatherInstructions(config)), + ); + } + + segments.push(section("Ballast", ballast)); + segments.push(section("Ask", ask.body)); + segments.push(`Write a single self-contained HTML file to: ${outputPath}`); + segments.push( + "Every prompt ends with: write a single self-contained HTML file to the output path given.", + ); + + const prompt = segments.filter(Boolean).join("\n\n"); + const inventory = inventoryFor({ ballast, brand, menu, ask: ask.body }); + return { prompt, inventory }; +} + +export function writePrompt( + config, + arm, + askN, + runK, + assembled = assemblePrompt(config, arm, askN, runK), +) { + const dir = join(config.out, arm, `ask-${askN}`); + mkdirSync(dir, { recursive: true }); + const promptPath = join(dir, `run-${runK}.prompt.md`); + const outputPath = join(dir, `run-${runK}.html`); + writeFileSync(promptPath, assembled.prompt, "utf8"); + writeFileSync( + join(dir, `run-${runK}.inventory.json`), + `${JSON.stringify(assembled.inventory, null, 2)}\n`, + "utf8", + ); + return { promptPath, outputPath, inventory: assembled.inventory }; +} + +export function dumpPackage(packageDir) { + const files = readdirSync(packageDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => entry.name) + .sort(); + return files + .map( + (file) => + `# ${basename(file)}\n\n${readFileSync(join(packageDir, file), "utf8").trim()}`, + ) + .join("\n\n"); +} + +function findAsk(config, askN) { + const asks = parseAsks(readFileSync(config.asks, "utf8")); + const ask = asks.find((item) => item.n === Number(askN)); + if (!ask) throw new Error(`ask ${askN} not found in ${config.asks}`); + return ask; +} + +function gatherInstructions(config) { + return [ + "Read the menu above against the ask.", + "Select only the Ghost node ids that are relevant.", + `Before generating, run: ${config.ghostBin} pull --package ${config.package}`, + "Use the pulled node bodies as steering context, then generate the artifact.", + ].join("\n"); +} + +function section(title, body) { + if (!body) return ""; + return `## ${title}\n\n${body}`; +} + +function firstLine(body) { + return ( + body + .split(/\r?\n/) + .find((line) => line.trim()) + ?.trim() ?? "" + ); +} + +function inventoryFor(parts) { + const inventory = { + ballast: countWords(parts.ballast), + brand: countWords(parts.brand), + menu: countWords(parts.menu), + ask: countWords(parts.ask), + }; + inventory.total = + inventory.ballast + inventory.brand + inventory.menu + inventory.ask; + inventory.tokensEstimate = Math.round(inventory.total * 1.33); + return inventory; +} + +function countWords(text) { + return (text.match(/\S+/gu) ?? []).length; +} + +function htmlPath(config, arm, askN, runK) { + return resolve(config.out, arm, `ask-${askN}`, `run-${runK}.html`); +} diff --git a/packages/steering-eval/lib/asks.mjs b/packages/steering-eval/lib/asks.mjs new file mode 100644 index 00000000..cc335c38 --- /dev/null +++ b/packages/steering-eval/lib/asks.mjs @@ -0,0 +1,49 @@ +export function parseAsks(markdown) { + const lines = markdown.split(/\r?\n/); + const asks = []; + let current = null; + + const flush = () => { + if (!current) return; + current.body = current.bodyLines.join("\n").trim(); + delete current.bodyLines; + asks.push(current); + }; + + for (const line of lines) { + const heading = line.match(/^##\s+Ask\s+(\d+)\s+[—-]\s+(.+)\s*$/u); + if (heading) { + flush(); + current = { + n: Number(heading[1]), + title: heading[2].trim(), + body: "", + bodyLines: [], + expect: [], + poison: [], + discount: [], + }; + continue; + } + + if (!current) continue; + + const meta = line.match(/^\s*(expect|poison|discount):\s*(.*?)\s*$/u); + if (meta) { + current[meta[1]] = splitIds(meta[2]); + continue; + } + + current.bodyLines.push(line); + } + + flush(); + return asks; +} + +function splitIds(value) { + return value + .split(",") + .map((part) => part.trim()) + .filter(Boolean); +} diff --git a/packages/steering-eval/lib/config.mjs b/packages/steering-eval/lib/config.mjs new file mode 100644 index 00000000..d4af02e5 --- /dev/null +++ b/packages/steering-eval/lib/config.mjs @@ -0,0 +1,134 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +const EXAMPLES = { + package: '"package": "./.ghost"', + asks: '"asks": "./asks.md"', + ballast: '"ballast": "./ballast.md"', + runsPerCell: '"runsPerCell": 5', + arms: '"arms": { "naked": true, "dump": true, "gather": true }', + out: '"out": "./out"', + ghostBin: '"ghostBin": "ghost"', + tells: '"tells": null', +}; + +export function loadConfig(dir = process.cwd()) { + const configPath = resolve(dir, "eval.config.json"); + let raw; + try { + raw = readFileSync(configPath, "utf8"); + } catch { + fail(`missing eval.config.json\nExample: ${EXAMPLES.package}`); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (error) { + fail(`invalid eval.config.json JSON: ${error.message}`); + } + + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + fail("invalid config: expected an object"); + } + + requireString(parsed, "package"); + requireString(parsed, "asks"); + requireString(parsed, "ballast"); + if ( + parsed.runsPerCell !== undefined && + !positiveInteger(parsed.runsPerCell) + ) { + invalid("runsPerCell"); + } + if ( + !parsed.arms || + typeof parsed.arms !== "object" || + Array.isArray(parsed.arms) + ) { + invalid("arms"); + } + for (const [arm, value] of Object.entries(parsed.arms)) { + if (!["naked", "dump", "gather", "dump-growth"].includes(arm)) { + fail(`invalid arms.${arm}: unknown arm\nExample: ${EXAMPLES.arms}`); + } + if ( + value !== true && + (typeof value !== "object" || value === null || Array.isArray(value)) + ) { + fail( + `invalid arms.${arm}: expected true or an options object\nExample: ${EXAMPLES.arms}`, + ); + } + if (arm === "dump-growth" && typeof value === "object" && value !== null) { + if ( + value.extraPackages !== undefined && + !stringArray(value.extraPackages) + ) { + fail( + 'invalid arms.dump-growth.extraPackages\nExample: "extraPackages": ["../other/.ghost"]', + ); + } + if (value.asks !== undefined && !value.asks.every?.(positiveInteger)) { + fail('invalid arms.dump-growth.asks\nExample: "asks": [1, 2]'); + } + } + } + if (parsed.out !== undefined && typeof parsed.out !== "string") + invalid("out"); + if (parsed.ghostBin !== undefined && typeof parsed.ghostBin !== "string") + invalid("ghostBin"); + if ( + parsed.tells !== undefined && + parsed.tells !== null && + typeof parsed.tells !== "string" + ) + invalid("tells"); + + const base = dirname(configPath); + const arms = { ...parsed.arms }; + if (arms["dump-growth"] && typeof arms["dump-growth"] === "object") { + arms["dump-growth"] = { + ...arms["dump-growth"], + extraPackages: (arms["dump-growth"].extraPackages ?? []).map((p) => + resolve(base, p), + ), + }; + } + + return { + ...parsed, + configPath, + configDir: base, + package: resolve(base, parsed.package), + asks: resolve(base, parsed.asks), + ballast: resolve(base, parsed.ballast), + runsPerCell: parsed.runsPerCell ?? 5, + arms, + tells: + parsed.tells === undefined || parsed.tells === null + ? null + : resolve(base, parsed.tells), + out: resolve(base, parsed.out ?? "./out"), + ghostBin: parsed.ghostBin ?? "ghost", + }; +} + +function requireString(obj, field) { + if (typeof obj[field] !== "string" || obj[field].trim() === "") + invalid(field); +} +function invalid(field) { + fail(`missing/invalid field: ${field}\nExample: ${EXAMPLES[field]}`); +} +function fail(message) { + throw new Error(`steering-eval config error: ${message}`); +} +function positiveInteger(value) { + return Number.isInteger(value) && value > 0; +} +function stringArray(value) { + return ( + Array.isArray(value) && value.every((item) => typeof item === "string") + ); +} diff --git a/packages/steering-eval/lib/report.mjs b/packages/steering-eval/lib/report.mjs new file mode 100644 index 00000000..9b4c0c16 --- /dev/null +++ b/packages/steering-eval/lib/report.mjs @@ -0,0 +1,264 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const ARM_ORDER = ["naked", "dump", "gather", "dump-growth"]; + +const TOKEN_NOTE = + "Token counts are estimates: words \u00d7 1.33, applied uniformly across arms."; + +const HONESTY_LINE = + "Every number below is deterministic \u2014 tell counts, selection-tape diffs, CSS extraction. No LLM judged anything."; + +function esc(s) { + return String(s) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; +} + +function fmt(n, digits = 2) { + if (n === null || n === undefined) return "\u2014"; + return Number(n) + .toFixed(digits) + .replace(/\.00$/, "") + .replace(/(\.\d)0$/, "$1"); +} + +function brandTokens(cell) { + const seg = cell.context?.segments ?? {}; + const pulled = cell.retrieval + ? Math.round((cell.retrieval.pulledWordsMean ?? 0) * 1.33) + : 0; + return (seg.brand ?? 0) + (seg.menu ?? 0) + pulled; +} + +async function fileExists(p) { + try { + await access(p); + return true; + } catch { + return false; + } +} + +// --- headline chart ------------------------------------------------------- + +function headlineChart(cells) { + const width = 700; + const height = 360; + const m = { top: 30, right: 40, bottom: 50, left: 60 }; + const plotW = width - m.left - m.right; + const plotH = height - m.top - m.bottom; + + // Aggregate per arm across asks. + const arms = []; + for (const arm of ARM_ORDER) { + const armCells = cells.filter((c) => c.arm === arm); + if (armCells.length === 0) continue; + const maxPossible = median(armCells.map((c) => c.scores.max_possible)); + const quality = maxPossible - median(armCells.map((c) => c.scores.median)); + const qMin = maxPossible - Math.max(...armCells.map((c) => c.scores.max)); + const qMax = maxPossible - Math.min(...armCells.map((c) => c.scores.min)); + const tokens = Math.round(median(armCells.map(brandTokens))); + arms.push({ arm, quality, qMin, qMax, tokens, maxPossible }); + } + + const logX = (t) => Math.log10(Math.max(t, 1)); + const maxLog = Math.max(...arms.map((a) => logX(a.tokens)), 1); + const xScale = (t) => m.left + (logX(t) / maxLog) * plotW; + const yMax = Math.max(...arms.map((a) => a.maxPossible)); + const yScale = (q) => m.top + plotH - (q / yMax) * plotH; + + const parts = []; + parts.push( + ``, + ); + // Axes + parts.push( + ``, + ``, + ); + // Y gridline labels + for (let q = 0; q <= yMax; q += Math.max(2, Math.ceil(yMax / 5))) { + const y = yScale(q); + parts.push( + `${q}`, + ); + } + // X axis label + parts.push( + `brand-context tokens (log scale, estimated: words \u00d7 1.33)`, + `quality (max \u2212 median tell score)`, + ); + // Points + whiskers + for (const a of arms) { + const x = xScale(a.tokens); + const yMed = yScale(a.quality); + const yLo = yScale(a.qMin); + const yHi = yScale(a.qMax); + parts.push( + ``, + ``, + ``, + ``, + `${esc(a.arm)}`, + `${a.tokens.toLocaleString("en-US")} tokens`, + ); + } + parts.push(""); + return parts.join("\n"); +} + +// --- per-ask sections ----------------------------------------------------- + +async function screenshotCell(outDir, cell, k) { + const rel = join(cell.arm, `ask-${cell.ask}`, `run-${k}.webp`); + const abs = join(outDir, rel); + if (await fileExists(abs)) { + const b64 = (await readFile(abs)).toString("base64"); + const uri = `data:image/webp;base64,${b64}`; + return `${esc(cell.arm)} ask ${cell.ask} run ${k}`; + } + return `
${esc(cell.arm)}
ask ${cell.ask} \u00b7 run ${k}
no screenshot
`; +} + +async function askSection(outDir, ask, askTitle, cells) { + const rows = []; + for (const cell of cells) { + const thumbs = []; + for (let k = 1; k <= cell.scores.n; k++) { + thumbs.push(await screenshotCell(outDir, cell, k)); + } + rows.push( + `
${esc(cell.arm)}
${thumbs.join("")}
`, + ); + } + + const tableRows = cells + .map((c) => { + const s = c.scores; + const con = c.consistency ?? {}; + return `${esc(c.arm)}${s.min} / ${s.median} / ${s.max}${fmt(con.accentHue)}${fmt(con.radius)}${fmt(con.fontStack)}${(c.context?.tokensEstimate ?? 0).toLocaleString("en-US")}`; + }) + .join("\n"); + + return `
+

Ask ${ask} \u2014 ${esc(askTitle)}

+
${rows.join("\n")}
+ + +${tableRows} +
armtell score min/med/maxaccent-hue consistencyradiusfontcontext tokens
+
`; +} + +// --- gather metrics table ------------------------------------------------- + +function gatherTable(cells) { + const gatherCells = cells.filter((c) => c.arm === "gather" && c.retrieval); + if (gatherCells.length === 0) return ""; + const rows = gatherCells + .map((c) => { + const r = c.retrieval; + const poison = + r.poisonPulls > 0 + ? `${r.poisonPulls}` + : `0`; + return `${c.ask} \u2014 ${esc(c.askTitle)}${poison}${fmt(r.precision)}${fmt(r.recall)}${fmt(r.stability)}${(r.pulledWordsMean ?? 0).toLocaleString("en-US")}`; + }) + .join("\n"); + return `
+

Gather metrics

+

From the selection tape. Poison pulls \u2014 wrong-register nodes pulled for the ask \u2014 are the legible number.

+ + +${rows} +
askpoison pullsprecisionrecallstabilitypulled words (mean)
+
`; +} + +// --- main ----------------------------------------------------------------- + +export async function renderReport(metricsJsonPath, outDir) { + const metrics = JSON.parse(await readFile(metricsJsonPath, "utf8")); + const cells = metrics.cells ?? []; + + const byAsk = new Map(); + for (const cell of cells) { + if (!byAsk.has(cell.ask)) byAsk.set(cell.ask, []); + byAsk.get(cell.ask).push(cell); + } + for (const list of byAsk.values()) { + list.sort((a, b) => ARM_ORDER.indexOf(a.arm) - ARM_ORDER.indexOf(b.arm)); + } + + const askSections = []; + for (const [ask, list] of [...byAsk.entries()].sort((a, b) => a[0] - b[0])) { + askSections.push(await askSection(outDir, ask, list[0].askTitle, list)); + } + + const html = ` + + + + +steering-eval report + + + +
+

steering-eval report

+

generated: ${esc(metrics.generatedAt ?? "unknown")}

+

package: ${esc(metrics.config?.package ?? "unknown")} \u00b7 runs per cell: ${metrics.config?.runsPerCell ?? "?"}

+

${HONESTY_LINE}

+
+
+

Quality vs. brand-context tokens

+

One point per arm (median of per-ask medians); whisker = min\u2013max band across all runs. Y is inverted median tell score, so higher is better.

+${headlineChart(cells)} +
+${askSections.join("\n")} +${gatherTable(cells)} +
+

Rebuild this report from out/ alone:

+

steering-eval shoot \u00b7 steering-eval score \u00b7 steering-eval report

+

${TOKEN_NOTE}

+
+ + +`; + + await mkdir(outDir, { recursive: true }); + const reportPath = join(outDir, "report.html"); + await writeFile(reportPath, html, "utf8"); + return reportPath; +} diff --git a/packages/steering-eval/lib/score.mjs b/packages/steering-eval/lib/score.mjs new file mode 100644 index 00000000..19fdbf1e --- /dev/null +++ b/packages/steering-eval/lib/score.mjs @@ -0,0 +1,291 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseAsks } from "./asks.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const defaultTellsPath = join(here, "..", "default-tells.json"); + +export function scoreHtml(html, tells, { discountTells = [] } = {}) { + const discounts = new Set(discountTells); + const hits = []; + let score = 0; + let max = 0; + for (const tell of tells) { + if (discounts.has(tell.id)) continue; + max += tell.weight; + const count = countTell(html, tell); + if (count > 0) { + hits.push({ id: tell.id, count }); + score += tell.weight; + } + } + return { score, max, hits }; +} + +export function scoreAll(config) { + const tells = JSON.parse( + readFileSync(config.tells ?? defaultTellsPath, "utf8"), + ).tells; + const asks = parseAsks(readFileSync(config.asks, "utf8")); + const cells = []; + for (const arm of enabledArms(config)) { + for (const ask of asksForArm(config, arm, asks)) { + const dir = join(config.out, arm, `ask-${ask.n}`); + const htmlFiles = existingRunFiles(dir, ".html"); + if (htmlFiles.length === 0) continue; + const runScores = htmlFiles.map( + (file) => + scoreHtml(readFileSync(join(dir, file), "utf8"), tells, { + discountTells: ask.discount, + }).score, + ); + const metas = existingRunFiles(dir, ".meta.json").map((file) => + JSON.parse(readFileSync(join(dir, file), "utf8")), + ); + const cell = { + arm, + ask: ask.n, + scores: summarize(runScores), + consistency: styleConsistency( + htmlFiles.map((file) => readFileSync(join(dir, file), "utf8")), + ), + context: { + tokensEstimate: mean( + metas + .map((meta) => meta.inventory?.tokensEstimate) + .filter(Number.isFinite), + ), + brandTokens: mean(metas.map((meta) => brandTokens(config, meta))), + }, + }; + if (arm === "gather") + cell.retrieval = retrievalMetrics(metas, ask, config); + cells.push(cell); + } + } + const metrics = { + generatedAt: new Date().toISOString(), + config: { runsPerCell: config.runsPerCell, package: config.package }, + cells, + }; + writeFileSync( + join(config.out, "metrics.json"), + JSON.stringify(metrics, null, 2), + "utf8", + ); + return metrics; +} + +export function retrievalMetrics(metas, ask, config) { + const expected = new Set(ask.expect ?? []); + const poison = new Set(ask.poison ?? []); + const runs = metas.map((meta) => new Set(meta.tape?.pulledIds ?? [])); + const perRun = runs.map((pulled) => { + const relevant = intersectionSize(pulled, expected); + return { + precision: + pulled.size > 0 ? relevant / pulled.size : expected.size === 0 ? 1 : 0, + recall: expected.size > 0 ? relevant / expected.size : 1, + poisonPulled: [...poison].some((id) => pulled.has(id)), + }; + }); + return { + precision: mean(perRun.map((run) => run.precision)), + recall: mean(perRun.map((run) => run.recall)), + poisonPulled: perRun.map((run) => run.poisonPulled), + poisonCount: perRun.filter((run) => run.poisonPulled).length, + selectionStability: meanPairwiseJaccard(runs), + pulledWords: config + ? mean( + metas.map((meta) => + (meta.tape?.pulledIds ?? []).reduce((sum, id) => { + const nodePath = join(config.package, `${id}.md`); + if (!existsSync(nodePath)) return sum; + return ( + sum + + (readFileSync(nodePath, "utf8").match(/\S+/gu) ?? []).length + ); + }, 0), + ), + ) + : 0, + }; +} + +function brandTokens(config, meta) { + const inv = meta.inventory ?? {}; + const brandWords = inv.brand ?? 0; + const menuWords = inv.menu ?? 0; + const pulledWords = (meta.tape?.pulledIds ?? []).reduce((sum, id) => { + const nodePath = join(config.package, `${id}.md`); + if (!existsSync(nodePath)) return sum; + return sum + (readFileSync(nodePath, "utf8").match(/\S+/gu) ?? []).length; + }, 0); + return Math.round((brandWords + menuWords + pulledWords) * 1.33); +} + +function countTell(html, tell) { + if (tell.kind === "text") + return tell.pattern ? html.split(tell.pattern).length - 1 : 0; + return [...html.matchAll(new RegExp(tell.pattern, "giu"))].length; +} + +function enabledArms(config) { + return Object.entries(config.arms) + .filter(([, value]) => value) + .map(([arm]) => arm); +} +function asksForArm(config, arm, asks) { + const allowed = config.arms[arm]?.asks; + return Array.isArray(allowed) + ? asks.filter((ask) => allowed.includes(ask.n)) + : asks; +} +function existingRunFiles(dir, suffix) { + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((file) => /^run-\d+/u.test(file) && file.endsWith(suffix)) + .sort(); +} +function summarize(values) { + if (values.length === 0) return { min: 0, median: 0, max: 0, mean: 0, n: 0 }; + const sorted = [...values].sort((a, b) => a - b); + return { + min: sorted[0], + median: median(sorted), + max: sorted.at(-1), + mean: mean(sorted), + n: sorted.length, + }; +} +function median(sorted) { + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} +function mean(values) { + return values.length + ? values.reduce((sum, value) => sum + value, 0) / values.length + : 0; +} +function intersectionSize(a, b) { + let n = 0; + for (const value of a) if (b.has(value)) n += 1; + return n; +} +function meanPairwiseJaccard(sets) { + if (sets.length < 2) return sets.length === 1 ? 1 : 0; + const scores = []; + for (let i = 0; i < sets.length; i += 1) { + for (let j = i + 1; j < sets.length; j += 1) { + const union = new Set([...sets[i], ...sets[j]]); + scores.push( + union.size ? intersectionSize(sets[i], sets[j]) / union.size : 1, + ); + } + } + return mean(scores); +} + +function styleConsistency(htmls) { + const dimensions = { + accentHue: htmls.map(dominantHue), + controlRadius: htmls.map((html) => + radiusFor( + html, + /(button|input|select|textarea|\.btn|\.button)[^{]*\{[^}]*\}/giu, + ), + ), + cardRadius: htmls.map((html) => + radiusFor(html, /(card|panel|surface|modal)[^{]*\{[^}]*\}/giu), + ), + fontStack: htmls.map(firstFontStack), + }; + return Object.fromEntries( + Object.entries(dimensions).map(([key, values]) => [ + key, + modalFraction(values.filter(Boolean)), + ]), + ); +} +function modalFraction(values) { + if (values.length === 0) return 0; + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return Math.max(...counts.values()) / values.length; +} +function firstFontStack(html) { + return ( + html + .match(/font-family\s*:\s*([^;}]+)/iu)?.[1] + .trim() + .toLowerCase() ?? null + ); +} +function radiusFor(html, selectorRegex) { + for (const block of html.matchAll(selectorRegex)) { + const radius = block[0] + .match(/border-radius\s*:\s*([^;}]+)/iu)?.[1] + .trim() + .toLowerCase(); + if (radius) return radius; + } + return null; +} +function dominantHue(html) { + const hues = []; + for (const color of extractColors(html)) { + const hsl = toHsl(color); + if (hsl && hsl.s >= 8) hues.push((Math.round(hsl.h / 30) * 30) % 360); + } + if (hues.length === 0) return null; + const counts = new Map(); + for (const hue of hues) counts.set(hue, (counts.get(hue) ?? 0) + 1); + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0][0]; +} +function extractColors(html) { + return [ + ...(html.match(/#[0-9a-f]{3,8}\b/giu) ?? []), + ...(html.match(/rgba?\([^)]*\)/giu) ?? []), + ...(html.match(/hsla?\([^)]*\)/giu) ?? []), + ]; +} +function toHsl(color) { + if (color.startsWith("#")) return rgbToHsl(...hexToRgb(color)); + const nums = color.match(/-?\d+(?:\.\d+)?/g)?.map(Number) ?? []; + if (color.startsWith("rgb")) return rgbToHsl(nums[0], nums[1], nums[2]); + if (color.startsWith("hsl")) + return { + h: ((nums[0] % 360) + 360) % 360, + s: nums[1] ?? 0, + l: nums[2] ?? 0, + }; + return null; +} +function hexToRgb(hex) { + let value = hex.slice(1); + if (value.length === 3 || value.length === 4) + value = value + .split("") + .map((c) => c + c) + .join(""); + const n = Number.parseInt(value.slice(0, 6), 16); + return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; +} +function rgbToHsl(r, g, b) { + r /= 255; + g /= 255; + b /= 255; + const max = Math.max(r, g, b), + min = Math.min(r, g, b); + let h = 0; + const l = (max + min) / 2; + const d = max - min; + const s = d === 0 ? 0 : d / (1 - Math.abs(2 * l - 1)); + if (d !== 0) { + if (max === r) h = 60 * (((g - b) / d) % 6); + else if (max === g) h = 60 * ((b - r) / d + 2); + else h = 60 * ((r - g) / d + 4); + } + return { h: (h + 360) % 360, s: s * 100, l: l * 100 }; +} diff --git a/packages/steering-eval/lib/shoot.mjs b/packages/steering-eval/lib/shoot.mjs new file mode 100644 index 00000000..16c86d6c --- /dev/null +++ b/packages/steering-eval/lib/shoot.mjs @@ -0,0 +1,52 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, readdirSync, statSync, unlinkSync } from "node:fs"; +import { resolve } from "node:path"; + +export function shootAll(config) { + requireTool("agent-browser", "screenshots"); + requireTool("cwebp", "screenshot optimization"); + + for (const html of findHtml(config.out)) { + const webp = html.replace(/\.html$/u, ".webp"); + if (existsSync(webp)) continue; + // agent-browser only emits PNG; shoot, convert to WebP, drop the PNG. + const png = html.replace(/\.html$/u, ".png"); + execFileSync("agent-browser", ["open", `file://${resolve(html)}`], { + stdio: "inherit", + }); + execFileSync("agent-browser", ["set", "viewport", "1280", "900"], { + stdio: "inherit", + }); + execFileSync("agent-browser", ["screenshot", png, "--full"], { + stdio: "inherit", + }); + execFileSync("cwebp", ["-quiet", "-q", "80", png, "-o", webp], { + stdio: "inherit", + }); + unlinkSync(png); + } +} + +function requireTool(name, purpose) { + const probe = spawnSync(name, ["-version"], { encoding: "utf8" }); + if (probe.error?.code === "ENOENT") { + console.error( + `steering-eval: ${name} is required for ${purpose}. Install ${name} and ensure it is on PATH.`, + ); + process.exit(1); + } +} + +function findHtml(root) { + const files = []; + function walk(dir) { + if (!existsSync(dir)) return; + for (const entry of readdirSync(dir)) { + const path = resolve(dir, entry); + if (statSync(path).isDirectory()) walk(path); + else if (/run-\d+\.html$/u.test(entry)) files.push(path); + } + } + walk(root); + return files.sort(); +} diff --git a/packages/steering-eval/lib/tape.mjs b/packages/steering-eval/lib/tape.mjs new file mode 100644 index 00000000..b13ff357 --- /dev/null +++ b/packages/steering-eval/lib/tape.mjs @@ -0,0 +1,75 @@ +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +export function openRun(config, arm, askN, runK) { + mkdirSync(config.out, { recursive: true }); + const lock = lockPath(config); + if (existsSync(lock)) { + console.error( + `steering-eval: run lock exists at ${lock}; finish or remove it before opening another run`, + ); + process.exit(1); + } + const tapeOffset = arm === "gather" ? tapeLength(config) : 0; + writeFileSync( + lock, + JSON.stringify( + { arm, askN: Number(askN), runK: Number(runK), tapeOffset }, + null, + 2, + ), + "utf8", + ); +} + +export function closeRun(config) { + const lock = lockPath(config); + if (!existsSync(lock)) throw new Error(`no open run lock at ${lock}`); + const run = JSON.parse(readFileSync(lock, "utf8")); + const events = + run.arm === "gather" ? readEventsSlice(config, run.tapeOffset ?? 0) : []; + unlinkSync(lock); + const pullEvents = events.filter( + (event) => event.event === "pull" && Array.isArray(event.ids), + ); + return { + events, + pulledIds: [...new Set(pullEvents.flatMap((event) => event.ids))], + pullCount: pullEvents.length, + }; +} + +function readEventsSlice(config, offset) { + const path = tapePath(config); + if (!existsSync(path)) return []; + const sliced = readFileSync(path).subarray(offset).toString("utf8"); + const events = []; + for (const line of sliced.split(/\r?\n/)) { + if (!line.trim()) continue; + try { + events.push(JSON.parse(line)); + } catch { + // Ignore partial or malformed tape lines from unrelated local use. + } + } + return events; +} + +function tapeLength(config) { + const path = tapePath(config); + return existsSync(path) ? statSync(path).size : 0; +} + +function tapePath(config) { + return join(config.package, ".events"); +} +function lockPath(config) { + return join(config.out, ".run.lock"); +} diff --git a/packages/steering-eval/package.json b/packages/steering-eval/package.json new file mode 100644 index 00000000..352d15eb --- /dev/null +++ b/packages/steering-eval/package.json @@ -0,0 +1,12 @@ +{ + "name": "@design-intelligence/steering-eval", + "private": true, + "type": "module", + "bin": { + "steering-eval": "./cli.mjs" + }, + "version": "0.1.0", + "scripts": { + "test": "vitest run" + } +} diff --git a/packages/steering-eval/test/report.test.ts b/packages/steering-eval/test/report.test.ts new file mode 100644 index 00000000..5f6ed82c --- /dev/null +++ b/packages/steering-eval/test/report.test.ts @@ -0,0 +1,73 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; + +// @ts-expect-error plain ESM module without type declarations +import { renderReport } from "../lib/report.mjs"; + +const FIXTURE = join(__dirname, "..", "fixtures", "metrics.sample.json"); + +const cleanups: string[] = []; + +async function freshOutDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), "steering-eval-report-")); + cleanups.push(dir); + return dir; +} + +function stripGeneratedAt(html: string): string { + return html + .split("\n") + .filter((line) => !line.includes("generated:")) + .join("\n"); +} + +afterAll(async () => { + await Promise.all( + cleanups.map((dir) => rm(dir, { recursive: true, force: true })), + ); +}); + +describe("renderReport", () => { + it("writes a self-contained report covering all arms", async () => { + const outDir = await freshOutDir(); + const reportPath = await renderReport(FIXTURE, outDir); + expect(reportPath).toBe(join(outDir, "report.html")); + + const html = await readFile(reportPath, "utf8"); + for (const arm of ["naked", "dump", "gather", "dump-growth"]) { + expect(html).toContain(arm); + } + }); + + it("renders the SVG headline chart", async () => { + const outDir = await freshOutDir(); + const html = await readFile(await renderReport(FIXTURE, outDir), "utf8"); + expect(html).toContain(' { + const outDir = await freshOutDir(); + const html = await readFile(await renderReport(FIXTURE, outDir), "utf8"); + expect(html).toContain('class="poison"'); + expect(html).toContain("#b91c1c"); + }); + + it("renders placeholders when no screenshots exist", async () => { + const outDir = await freshOutDir(); + const html = await readFile(await renderReport(FIXTURE, outDir), "utf8"); + expect(html).toContain("placeholder"); + expect(html).toContain("no screenshot"); + expect(html).not.toContain("data:image/webp;base64"); + }); + + it("is deterministic modulo the generatedAt line", async () => { + const dirA = await freshOutDir(); + const dirB = await freshOutDir(); + const htmlA = await readFile(await renderReport(FIXTURE, dirA), "utf8"); + const htmlB = await readFile(await renderReport(FIXTURE, dirB), "utf8"); + expect(stripGeneratedAt(htmlA)).toBe(stripGeneratedAt(htmlB)); + }); +}); diff --git a/packages/steering-eval/test/steering-eval.test.ts b/packages/steering-eval/test/steering-eval.test.ts new file mode 100644 index 00000000..642aa337 --- /dev/null +++ b/packages/steering-eval/test/steering-eval.test.ts @@ -0,0 +1,149 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { assemblePrompt } from "../lib/arms.mjs"; +import { parseAsks } from "../lib/asks.mjs"; +import { loadConfig } from "../lib/config.mjs"; +import { retrievalMetrics, scoreHtml } from "../lib/score.mjs"; +import { closeRun, openRun } from "../lib/tape.mjs"; + +let dir: string; +beforeEach(() => { + dir = join( + tmpdir(), + `steering-eval-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + mkdirSync(dir, { recursive: true }); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +function writeProject() { + const pkg = join(dir, ".ghost"); + mkdirSync(pkg, { recursive: true }); + writeFileSync( + join(pkg, "manifest.yml"), + "schema: ghost.fingerprint-package/v1\nid: test\n", + ); + writeFileSync(join(pkg, "glossary.md"), "Glossary prose\n"); + writeFileSync( + join(pkg, "principle.test.md"), + "---\ndescription: Test\n---\n\nNode prose here.\n", + ); + writeFileSync( + join(dir, "asks.md"), + "## Ask 1 — page\n\nMake a page.\nexpect: principle.test\npoison: pattern.email\ndiscount: font-x\n", + ); + writeFileSync(join(dir, "ballast.md"), "ballast words\n"); + writeFileSync( + join(dir, "eval.config.json"), + JSON.stringify({ + package: "./.ghost", + asks: "./asks.md", + ballast: "./ballast.md", + arms: { naked: true, dump: true, gather: true }, + ghostBin: "node", + }), + ); + return loadConfig(dir); +} + +describe("steering-eval", () => { + it("teaches config validation errors", () => { + writeFileSync( + join(dir, "eval.config.json"), + JSON.stringify({ asks: "./asks.md", ballast: "./ballast.md", arms: {} }), + ); + expect(() => loadConfig(dir)).toThrow(/package.*Example/s); + }); + + it("parses asks and strips retrieval lines", () => { + const asks = parseAsks( + "## Ask 2 — email\n\nBody line.\nexpect: a, b\npoison: c\ndiscount: font\nMore body.\n", + ); + expect(asks[0]).toMatchObject({ + n: 2, + title: "email", + expect: ["a", "b"], + poison: ["c"], + discount: ["font"], + }); + expect(asks[0].body).toBe("Body line.\nMore body."); + }); + + it("assembles naked and dump prompts", () => { + const config = writeProject(); + const naked = assemblePrompt(config, "naked", 1, 1).prompt; + expect(naked).not.toContain("Node prose here"); + const dump = assemblePrompt(config, "dump", 1, 1).prompt; + expect(dump).toContain("Node prose here"); + }); + + it("assembles gather prompts with menu output", () => { + const config = writeProject(); + const script = join(dir, "fake-ghost.mjs"); + writeFileSync(script, "console.log('MENU marker principle.test')\n"); + config.ghostBin = process.execPath; + config.ghostArgs = [script]; + const prompt = assemblePrompt(config, "gather", 1, 1).prompt; + expect(prompt).toContain("MENU marker"); + expect(prompt).toContain("pull "); + }); + + it("slices the tape from lock offset", () => { + const config = writeProject(); + writeFileSync( + join(config.package, ".events"), + `${JSON.stringify({ ts: "old", event: "pull", ids: ["old"] })}\n`, + ); + openRun(config, "gather", 1, 1); + writeFileSync( + join(config.package, ".events"), + `${readFileSync(join(config.package, ".events"), "utf8")}${JSON.stringify({ ts: "new", event: "gather", menu: ["a"] })}\n${JSON.stringify({ ts: "new", event: "pull", ids: ["a", "b"] })}\n`, + ); + const closed = closeRun(config); + expect(closed.pulledIds.sort()).toEqual(["a", "b"]); + expect(closed.pullCount).toBe(1); + expect(closed.events).toHaveLength(2); + }); + + it("hard-fails when a lock exists", () => { + const cli = resolve("packages/steering-eval/cli.mjs"); + writeProject(); + mkdirSync(join(dir, "out"), { recursive: true }); + writeFileSync(join(dir, "out", ".run.lock"), "{}\n"); + const result = spawnSync( + process.execPath, + [cli, "prompt", "naked", "1", "--run", "1"], + { cwd: dir, encoding: "utf8" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain("run lock exists"); + }); + + it("scores known tells", () => { + const tells = [ + { id: "a", kind: "text", pattern: "Hello", weight: 2 }, + { id: "b", kind: "regex", pattern: "color:\\s*red", weight: 3 }, + { id: "c", kind: "regex", pattern: "border", weight: 1 }, + ]; + expect( + scoreHtml("Hello ", tells).score, + ).toBe(6); + }); + + it("computes retrieval math", () => { + const metrics = retrievalMetrics( + [ + { tape: { pulledIds: ["a", "b"] } }, + { tape: { pulledIds: ["a", "c"] } }, + ], + { expect: ["a", "b"], poison: ["c"] }, + ); + expect(metrics.precision).toBe(0.75); + expect(metrics.recall).toBe(0.75); + expect(metrics.poisonCount).toBe(1); + expect(metrics.selectionStability).toBeCloseTo(1 / 3); + }); +}); diff --git a/packages/vessel-light/.ghost/.gitignore b/packages/vessel-light/.ghost/.gitignore new file mode 100644 index 00000000..be73df2f --- /dev/null +++ b/packages/vessel-light/.ghost/.gitignore @@ -0,0 +1 @@ +.events diff --git a/packages/vessel-light/.ghost/anti-goal.median.md b/packages/vessel-light/.ghost/anti-goal.median.md new file mode 100644 index 00000000..a0a5cefb --- /dev/null +++ b/packages/vessel-light/.ghost/anti-goal.median.md @@ -0,0 +1,86 @@ +--- +description: "The model's median defaults this fingerprint refuses — gather for any greenfield visual surface or first-draft copy. Each rule is reject→replace; delete lines your brand legitimately violates." +--- + +This is the model's median, not your brand. Each rule is reject→replace. +Delete every line your brand legitimately violates — `ghost validate` will +warn on any check reference the deletion orphans — delete the paired flag too. + +These are not aesthetic opinions. Where a count is given, it is the measured +convergence of 300 unsteered generations across three frontier models (the +antimedian experiment): the defaults a model reaches for when nobody hands it +a brand. An output showing several of these tells reads as generated, +whatever else it does right. A deterministic floor rides alongside these +rules in the paired check — mechanical tells like gradient-filled text, +emoji as interface icons, em-dash pileups, and aphoristic cadence — +verified at review, never steered in prose. + +### Hover-lift +Reject hover-lift (`translateY` + growing shadow) as the default interaction +→ confirm with color and background change at the fingerprint's fast +duration. (measured: 341) + +### Indigo accent +Reject the indigo/blue/purple default accent (`#4f46e5`, `#2563eb`, +`#8b5cf6` family; purple/violet hue 260–310) → the fingerprint's declared +palette; absent one, monochrome plus a single functional accent. + +### Dark theme +Reject unprompted dark theme → the fingerprint's declared surface. +Dark-to-look-cool and light-to-be-safe are the same retreat from a decision. +(measured: 271) + +### Gradients +Reject gradient page and section backgrounds and gradient-filled CTAs → flat +surfaces in semantic roles. + +### Side-stripe +Reject side-stripe borders (a thick colored border on one side of a rounded +card) — the single most recognizable tell of AI-generated UI → a full +hairline border, a 4–8% surface tint, or a leading glyph. + +### Cream surface +Reject the cream/sand/beige default surface (warm off-white; token names +like `--cream`, `--sand`, `--parchment` are the tell) → a true off-white, +the brand's own hue, or a committed color; warmth via accent and type, not +the ground. + +### Glassmorphism +Reject glassmorphism (decorative backdrop blur) → flat surfaces with real +borders and the fingerprint's elevation tiers. + +### Chat bubbles +Reject chat bubbles with initials-circle avatars for assistant turns → +assistant text plain on the page surface; user turns marked compactly. + +### Stock copy +Reject stock template copy ("Simple, transparent pricing") → headings that +state what this product specifically does. + +### Celebration +Reject celebration copy and UI (exclamation success, confetti language) → +quiet factual confirmation. + +### Nested cards +Reject nested cards and everything-in-cards → one surface level per region; +interior hierarchy from spacing and type; spacing and alignment group +without card overhead. + +### Hero metric +Reject the hero-metric template (big number + small label + supporting stats +as default proof) → evidence specific to the product, or nothing. A +prominent metric showing real user data is fine. + +### Every button primary +Reject every button a primary button → one primary per view; the rest step +down the fingerprint's emphasis ladder. + +### Eyebrow kicker +Reject an eyebrow kicker on every section → at most one, where the register +sanctions it; one named kicker is voice, every-section is AI grammar. + +### Decorative motion +Reject decorative looping animation (pulse/float/shimmer) and uniform +fade-and-rise on every scrolled section → motion as evidence of state +change; loops only for genuine loading; decoration never compensates for +weak structure — fix spacing, copy, and hierarchy first. diff --git a/packages/vessel-light/.ghost/anti-goal.tells.md b/packages/vessel-light/.ghost/anti-goal.tells.md new file mode 100644 index 00000000..73ed11c8 --- /dev/null +++ b/packages/vessel-light/.ghost/anti-goal.tells.md @@ -0,0 +1,37 @@ +--- +description: "Failure modes specific to this brand's own signature — gather alongside any signature node you pull." +--- + +These are the near-misses of Vessel's own signature: outputs that got close +enough to cite the brand and still missed it. Each reject names its +replacement. + +Reject rectangular buttons. A rectangular button is not a variant; it is a +different design system → controls take `--radius-control`, the pill. + +Reject swapped radius roles: the surface radius on a control, the pill on a +surface, or 20px cells in a data table → `--radius-control` on what you +click, `--radius-surface` on what contains, small radius on data cells. + +Reject expression hues on interactive controls, an invented sixth hue, or a +tinted gray posing as monochrome → the five named expression hues at their +register's volume, and a true gray spine. An invented hue is not a bolder +Vessel; it is a different brand. + +Reject custom shadows that sit near a tier → the exact tier — card, popover, +or modal — chosen by interaction importance. + +Reject borrowing `--shadow-btn` or `--shadow-kbd` for layout → component +shadows belong to the primitives that own them; layout elevation comes from +the three tiers. + +Reject editorial display type in product UI, and product type timidity in +editorial → each register keeps its own vocabulary; a modal title is not a +poster, and a hero at product scale reads as timid. + +Reject borders around assistant text → the assistant speaks as plain text on +the page surface; use space, tone, and type hierarchy instead. + +Fidelity note: pills and fluid clamp display headings are medians Vessel +deliberately shares — their presence is fidelity, not drift. Convergence is +not the crime; surrendering the choice is. diff --git a/packages/vessel-light/.ghost/checks/conversation-grammar.md b/packages/vessel-light/.ghost/checks/conversation-grammar.md new file mode 100644 index 00000000..938e77ff --- /dev/null +++ b/packages/vessel-light/.ghost/checks/conversation-grammar.md @@ -0,0 +1,20 @@ +--- +name: Conversation grammar +description: Flags conversation UI that breaks plain assistant text, prompt-input structure, primary-action discipline, or collapsed tool output. +severity: high +references: + - grammar.conversation + - grammar.surfaces +--- + +Apply this check to diffs that touch AI threads, agent consoles, prompt composers, chat messages, or tool-call rendering. + +Flag assistant messages wrapped in bordered, filled, elevated, or card-like surfaces. Assistant turns should sit as plain text on the page surface. + +Flag user messages that are not compact muted surfaces aligned to the right when the layout has a conversational thread. + +Flag controls placed inside the prompt-input textarea region. The textarea is for writing only; controls belong in the single row below it. + +Flag more than one primary button in the input row. Send, stop, retry, attach, model select, and tools must not compete as primary actions. + +Flag tool output shown expanded by default. The default state is a one-line labeled summary with status; mono detail appears after expansion. diff --git a/packages/vessel-light/.ghost/checks/median-tells.md b/packages/vessel-light/.ghost/checks/median-tells.md new file mode 100644 index 00000000..c15e44e5 --- /dev/null +++ b/packages/vessel-light/.ghost/checks/median-tells.md @@ -0,0 +1,103 @@ +--- +name: Median tells +description: Flags the measured defaults of unsteered generation and current model-signature tells — hover-lift, default accents, unprompted dark theme, gradient text, frequency tells, and per-model signatures. +severity: high +references: + - anti-goal.median > Hover-lift + - anti-goal.median > Indigo accent + - anti-goal.median > Dark theme + - anti-goal.median > Gradients + - anti-goal.median > Glassmorphism + - anti-goal.median > Side-stripe + - anti-goal.median > Cream surface + - anti-goal.median > Chat bubbles + - anti-goal.median > Stock copy + - anti-goal.median > Celebration + - anti-goal.median > Hero metric + - anti-goal.median > Eyebrow kicker +--- + +This is Vessel's adaptation of the shared median check; the fidelity carve-outs +below are the body's own. These flags target the measured convergence patterns +of unsteered model generation, plus tells specific to individual models. Each +is mechanically detectable in a diff. Pruning a rule from `anti-goal.median` +orphans its paired reference here — `ghost validate` warns; delete the flag and +its reference together. + +Flag `transform` with `translateY` inside a `:hover` rule on cards, buttons, +or list items, especially paired with a shadow increase. Hover confirmation +in Vessel is color and background change, not lift. +(`anti-goal.median > Hover-lift`) + +Flag accent values in the indigo/blue/purple default family (`#4f46e5`, +`#6366f1`, `#2563eb`, `#3b82f6`, `#8b5cf6`, and close neighbors) unless the +diff shows the user asked for them. They are model defaults, not palette +members. (`anti-goal.median > Indigo accent`) + +Flag whole-page dark backgrounds when the ask did not request dark mode. +Dark surfaces in Vessel are the editorial dark band or an explicit `.dark` +theme, never an unprompted default. (`anti-goal.median > Dark theme`) + +Flag `linear-gradient` or `radial-gradient` as page or section backgrounds, +and gradient-filled buttons. (`anti-goal.median > Gradients`) + +Flag `backdrop-filter: blur` used for glassmorphism cards. +(`anti-goal.median > Glassmorphism`) + +Flag `background-clip: text` (with or without the `-webkit-` prefix) paired +with a gradient. Emphasis comes from weight or size in a single solid color. + +Flag a thick colored border on one side of an element (`border-left` or a +`border-l-*` utility at 2px or more in a non-neutral color) while the other +sides stay thin. (`anti-goal.median > Side-stripe`) + +Flag warm off-white page backgrounds in the cream/sand/beige band, and token +names like `--cream`, `--sand`, `--parchment`, `--linen` introduced by the +diff. (`anti-goal.median > Cream surface`) + +Flag assistant messages rendered as bubbles with initials-circle avatars. +(`anti-goal.median > Chat bubbles`) + +Flag emoji used as icons or imagery in interface chrome. Text labels carry +meaning. + +Flag stock template copy in headings: "Simple, transparent pricing", +"Welcome back", and interchangeable-with-a-competitor phrasing. Recommend +copy that states what this product specifically does. +(`anti-goal.median > Stock copy`) + +Flag exclamation-marked success copy, confetti language, and celebratory UI +("You did it!", "Awesome!"). Confirmation is quiet and factual. +(`anti-goal.median > Celebration`) + +Flag the hero-metric template — a big number, small label, and supporting +stats as default proof — unless the metric shows real user data. Recommend +evidence specific to the product, or nothing. +(`anti-goal.median > Hero metric`) + +Frequency tells — the crime is repetition, not the move (advisory): + +Flag three or more uppercase, tracked eyebrow kickers above section headings +in one page. One named kicker is voice; a kicker on every section is model +grammar. (`anti-goal.median > Eyebrow kicker`) + +Flag five or more em-dashes in body copy in one view. + +Flag three or more instances of the aphoristic rebuttal cadence ("Not X. Y." +/ "Sentence. No qualifiers.") in one page's copy. + +Model-signature tells — skip any block whose model did not produce the diff +(advisory): + +Codex: flag a 1px border paired with a box-shadow of 16px blur or more on +the same element — pick a solid border or a tight shadow, not both. Flag +border-radius of 32px or more on cards, sections, or inputs. Flag 1px +linear-gradient grid or repeating-stripe backgrounds used as decoration. + +Gemini: flag `transform` (scale, rotate, translate) on `img` inside a +`:hover` rule, including group-hover utilities targeting a child image. +Animate the card's background, border, or shadow instead. + +Do not flag pill-shaped controls or fluid clamp display headings — those +median patterns are deliberately shared by Vessel and their presence is +fidelity. diff --git a/packages/vessel-light/.ghost/checks/motion-restraint.md b/packages/vessel-light/.ghost/checks/motion-restraint.md new file mode 100644 index 00000000..52040bfb --- /dev/null +++ b/packages/vessel-light/.ghost/checks/motion-restraint.md @@ -0,0 +1,20 @@ +--- +name: Motion restraint +description: Flags non-token motion, looping decoration, and keyframes that do not explain state change. +severity: medium +references: + - grammar.motion + - signature.temperature +--- + +Review changed transitions and animations for vocabulary first. + +Flag durations that do not use `--duration-fast`, `--duration-normal`, or `--duration-slow`. A hard-coded millisecond value is a drift even when it matches the token today. + +Flag custom easing unless it uses the fingerprint's one ease or a browser default already required by the primitive. + +Flag looping animations outside explicit loading states. Loading must be tied to ongoing work, not ambient motion. + +Flag decorative keyframes: glow, float, shimmer, bounce, pulse, parallax, or any motion that can be removed without making a state change harder to understand. + +Recommend the smallest opacity or transform transition that explains the state change. diff --git a/packages/vessel-light/.ghost/checks/relationships.md b/packages/vessel-light/.ghost/checks/relationships.md new file mode 100644 index 00000000..1dfe6391 --- /dev/null +++ b/packages/vessel-light/.ghost/checks/relationships.md @@ -0,0 +1,33 @@ +--- +name: Relationship discipline +description: Flags structural violations of the grammar — emphasis-ladder breaks, sibling margins, decorative borders, nested cards. These rules survive any adaptation. +severity: high +references: + - grammar.hierarchy + - grammar.rhythm + - grammar.surfaces + - anti-goal.median > Nested cards +--- + +These assertions test relationships between token roles, not the values behind +them. They hold for any brand built on this grammar; an adapted package keeps this check +unchanged. + +Flag more than one primary-variant button per view. Secondary actions step +down the emphasis ladder: secondary, outline, ghost, or link. + +Flag `margin` set on siblings inside a stack in changed CSS or inline styles. +Rhythm comes from the gap scale; recommend changing the stack gap or splitting +the stack. + +Flag borders, fills, gradients, glass effects, or colored accents used to make +ordinary structure feel designed. Recommend semantic surfaces, stack spacing, +text tone, or the existing elevation tier instead. + +Flag card-like surfaces nested inside card-like surfaces. One surface level +per region; interior hierarchy comes from spacing and type, not another +container. + +Flag `box-shadow` values that belong to no declared elevation tier and no +component-owned shadow. Do not accept a custom shadow because it is visually +close to a tier — elevation is a closed set, whatever the values are. diff --git a/packages/vessel-light/.ghost/checks/values.md b/packages/vessel-light/.ghost/checks/values.md new file mode 100644 index 00000000..8587251c --- /dev/null +++ b/packages/vessel-light/.ghost/checks/values.md @@ -0,0 +1,51 @@ +--- +name: Value discipline +description: Flags off-signature values — non-pill controls, off-palette hues, raw color literals, expression over budget. Adapting the dials rewrites this check alongside the signature nodes. +severity: high +references: + - signature.shape + - signature.palette + - grammar.color-roles + - register.data-density + - register.editorial + - register.email +--- + +These assertions test Vessel's current answers to the signature dials. A +adaptation that changes the dials rewrites this check with them; the paired +relationship check stays. + +Review changed HTML and CSS by view, not just by file. Classify the register +of each changed view first: table layout, `bgcolor`, or a 600px wrapper means +email; display-scale headings, `--section-padding-vertical`, or +`--surface-dark-*` means editorial; compressed smallest-gap-step layouts with +mono numerals means data-density; otherwise product. + +Flag buttons or text inputs that are not pill-radius (`--radius-control`). +A rectangular control is a token violation, not a style choice. + +Flag surfaces that carry a radius other than `--radius-surface`, and radius +roles swapped across the boundary: control radius on a surface, surface +radius on a control. + +Count distinct non-gray colors visible in each view, then judge against that +register's expression budget: product views get one functional accent, with +expression hues permitted only inside chart or visualization markup; data +consoles get one status hue family with the same chart carve-out; email gets +one expressive moment in one hue; editorial pages get at most two expression +hues. Flag anything over budget, and flag expression hues used +atmospherically outside their register. Status colors count toward the total +unless the view compares two or more distinct statuses side by side. + +Flag any color that is neither a gray, a status token, nor one of the five +expression hues (`--expression-1` through `--expression-5`, or their hex +values in email). Invented hues are drift in every register. + +Flag expression hues on buttons, inputs, or links in every register. Controls +stay monochrome; no budget covers them. + +Flag raw hex, rgb, hsl, or named color values in changed CSS or inline styles +when a Vessel token could express the same role. The authoring vocabulary is +semantic tokens, not literal color. Exception: email files — hardcoded values +transcribed from tokens are the contract there; flag only values that +correspond to no token and no expression hue. diff --git a/packages/vessel-light/.ghost/glossary.md b/packages/vessel-light/.ghost/glossary.md new file mode 100644 index 00000000..8017688a --- /dev/null +++ b/packages/vessel-light/.ghost/glossary.md @@ -0,0 +1,45 @@ +--- +kinds: + - name: grammar + - name: signature + - name: register + - name: anti-goal + posture: guard +--- + +# grammar + +The brand's decision logic: closed sets, role vocabularies, and assembly +rules, stated in token roles and never in literal values. Grammar survives an +adaptation unchanged — adapt the package, swap every value, and these nodes still +hold. Gather grammar before inventing structure. + +# signature + +The dials: the choices that make this brand this brand, each stated as a +current answer that stands until you replace it. Signature nodes name real +values because they are the values — on adaptation, edit the token roles in +`materials/tokens.css` and restate the node. Gather signature before setting +any value a dial governs. + +# register + +A condition-scoped contract: a situation — data-dense consoles, editorial +pages, transactional email — where parts of the default rules invert. Each +register names its condition first. Truths from the wrong register are +poison, not context; gather a register only when its condition matches the +task. + +# anti-goal + +What this package refuses, each reject paired with its replacement. +`anti-goal.median` is the model's floor, not the brand's taste; +`anti-goal.tells` is the near-miss map of this brand's own signature. Gather +anti-goals before styling anything greenfield. + +--- + +To adapt this package to another brand, follow the `adapting-a-starter` +recipe in the ghost skill bundle. In short: new manifest id, prune the +median rules your brand legitimately violates, answer each signature node, +edit `materials/tokens.css`, regenerate the refs, rewrite the tells. diff --git a/packages/vessel-light/.ghost/grammar.color-roles.md b/packages/vessel-light/.ghost/grammar.color-roles.md new file mode 100644 index 00000000..01f24754 --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.color-roles.md @@ -0,0 +1,30 @@ +--- +description: "Semantic color roles and the rules that govern them — gather before choosing any color; roles not raw values, status is meaning, expression is register-gated and never on controls." +materials: + - materials/tokens.css + - "**/*.css" + - "**/*.html" +--- + +The token file is the source of truth. An agent may combine tokens, but it +may not author around them. Raw color values are implementation detail, never +product language. + +Author with semantic roles: `background`, `foreground`, `card`, `popover`, +`primary`, `secondary`, `muted`, `accent`, `border`, `input`, `ring`, and the +status roles. If a container needs a color, it first needs a role. + +The status roles — destructive, success, warning, info — exist only when +meaning demands them. Destructive means destructive or error. Success means +success. Warning means warning. Info means information. None of them are +brand accents, and they never moonlight as atmosphere, in any register. + +One view should not perform a color palette. If a status color is present, +let the rest of the view stay on the base roles. Richness beyond this is +register-gated: a closed expression set (`--expression-*`) exists, but its +size, members, and volume ladder are a brand answer — see the palette +signature — and each register caps how loud they may be. + +The constant that holds across every register: expression never touches what +you click. Buttons, inputs, and links stay on the base roles everywhere. A +colored control is a different design system. diff --git a/packages/vessel-light/.ghost/grammar.conversation.md b/packages/vessel-light/.ghost/grammar.conversation.md new file mode 100644 index 00000000..980a8ac3 --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.conversation.md @@ -0,0 +1,30 @@ +--- +description: "Conversation grammar — plain assistant text, compact user surfaces, collapsed tool calls, one structured prompt input; gather for any AI thread, agent console, review assistant, or prompt composer." +materials: + - materials/ref/composition.conversation.html + - "**/*.html" +--- + +Conversation UI is not chat cosplay. The assistant speaks on the page surface +as plain text: no bubble, no border, no fill. + +User turns are compact muted surfaces aligned right. They mark authorship +without turning the thread into alternating balloons. + +Assistant hierarchy comes from prose, spacing, and type. Wrapping assistant +messages in cards makes the system look defensive and wastes density. + +Tool calls are operational evidence. Collapse them to a labeled one-line +summary with status. Expand only when the user asks for detail, then show +mono content inside the disclosed area. + +The prompt input is one bordered surface. The textarea region stays empty of +controls so writing remains the focus. Attachments, model choices, secondary +tools, and send live in a single row below it. + +There is one primary send action. Stop and send are mutually exclusive states +of the same action area, not two competing primary buttons. + +Use the conversation reference when building any AI thread, agent console, +review assistant, or prompt composer. It carries the grammar that agents most +often get wrong. diff --git a/packages/vessel-light/.ghost/grammar.hierarchy.md b/packages/vessel-light/.ghost/grammar.hierarchy.md new file mode 100644 index 00000000..db66170f --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.hierarchy.md @@ -0,0 +1,48 @@ +--- +description: "The closed hierarchy vocabulary — six text variants, seven tones, a five-rung control emphasis ladder, one primary per view — gather for any view that contains text or actions." +materials: + - materials/primitives.css + - materials/ref/composition.form.html + - "**/*.html" + - "**/*.css" +--- + +Hierarchy is a closed vocabulary, not a size slider. Every piece of text and +every control picks from a named set; anything outside the set is a broken +primitive, not a variant. + +The text variants are exactly six: display, headline, title, body, label, +mono. Variant names are jobs, not decoration. Display leads a rare editorial +moment. Headline names a section. Title anchors a card, dialog, or compact +region. Body carries reading. Label names structure — category tags, field +labels, bylines, metadata, compact status. Mono carries code and machine +detail. Do not fake hierarchy with arbitrary font sizes; choose the tier that +matches the job, then use tone, weight, and spacing for the rest. + +The tones are exactly seven: default, muted, inverse, success, warning, info, +destructive. Tone is part of the message. Default is the main reading plane. +Muted carries secondary information. Inverse is for dark or primary surfaces. +The four status tones appear only when the words have that state meaning. +Hierarchy starts with tone and weight before size — a secondary note usually +wants muted body or label, not a smaller custom font. If the prose needs +emphasis, improve the sentence before adding a style. + +The control emphasis ladder is exactly five rungs: primary, secondary, +outline, ghost, link. One primary action per view is the rule; everything +else steps down the ladder. Primary is for the action the screen exists to +complete — if two buttons both look primary, the hierarchy failed. +Destructive is a meaning, not a rung: it is rare, it names the destructive +act directly, and it is never borrowed for urgency, emphasis, or brand heat. + +Fields are quiet until they are active: hairline border, muted placeholder, +clear label, and a focused ring when the user engages. The focus ring is +guidance, not decoration. Errors state facts next to the field that caused +them — do not hide field errors in modals, toasts, or generic banners when +the user needs to fix one input. + +Balanced text is for headings and compact statements. Do not balance long +body copy into jagged reading. + +The form reference shows the intended decision order: stacked labels and +fields, compact helper text, one submit, secondary escape. Copy the decision +order before adjusting surface detail. diff --git a/packages/vessel-light/.ghost/grammar.motion.md b/packages/vessel-light/.ghost/grammar.motion.md new file mode 100644 index 00000000..d7cbfbfd --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.motion.md @@ -0,0 +1,27 @@ +--- +description: "Motion doctrine — motion is evidence of state change, never decoration; a closed vocabulary of three duration roles and one ease; gather for any transition, animation, or hover treatment." +materials: + - materials/tokens.css + - "**/*.css" + - "**/*.html" +--- + +Motion is evidence of a state change. It confirms hover, press, reveal, +collapse, entrance, exit, and spatial movement. It does not entertain. + +The entire vocabulary is three duration roles and one ease. Fast is for hover +and press. Normal is for reveals, fades, and small state changes. Slow is +reserved for spatial transitions where the user needs to understand movement. +Do not introduce novelty easings because a surface feels static; the ease's +character is a brand answer — see the temperature signature. + +Nothing loops except explicit loading states. A spinner may continue because +work continues. Decorative pulsing, floating, glowing, and attention-seeking +keyframes are off-language. + +Prefer opacity and small transform changes. If removing an animation does not +reduce comprehension, the animation was decoration. + +Condition: marketing and editorial surfaces may stage entrances — scroll reveals and +section transitions are part of editorial rhythm, still built from the three +durations and the one ease. In product UI the same staging is decoration. diff --git a/packages/vessel-light/.ghost/grammar.rhythm.md b/packages/vessel-light/.ghost/grammar.rhythm.md new file mode 100644 index 00000000..afdc9f1b --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.rhythm.md @@ -0,0 +1,27 @@ +--- +description: "Layout rhythm — all layout is stacks with a closed gap step set; gather before laying anything out; never ad-hoc sibling margins." +materials: + - materials/primitives.css + - "**/*.html" + - "**/*.css" +--- + +All layout is stacks. Rhythm comes from relationships between siblings, not +from isolated margins pasted onto whichever element was last touched. + +The gap steps are exactly five: `--gap-xs`, `--gap-sm`, `--gap-md`, +`--gap-lg`, `--gap-xl`. Choose the gap that states the relationship. Do not +tune by single pixels to make a screenshot pass. + +The default is column, medium gap, stretch alignment, start justification. +That is the ordinary reading rhythm. + +Columns are for almost everything: forms, cards, message lists, modal bodies, +settings, empty states, and page sections. A column lets the user scan. + +Rows are conditional. Use them for controls, metadata lines, compact status, +and paired label/value moments. If a row starts wrapping awkwardly, it +probably wanted to be a column. + +Ad-hoc margins between siblings are forbidden because they hide the rhythm. +When spacing feels wrong, change the stack gap or split the stack. diff --git a/packages/vessel-light/.ghost/grammar.surfaces.md b/packages/vessel-light/.ghost/grammar.surfaces.md new file mode 100644 index 00000000..b9a5e9b3 --- /dev/null +++ b/packages/vessel-light/.ghost/grammar.surfaces.md @@ -0,0 +1,35 @@ +--- +description: "Surface roles and the closed elevation set — gather for any card, popover, modal, dialog, scrim, or bordered container; flat is the default, borders are structural, exactly three elevation tiers." +materials: + - materials/primitives.css + - materials/ref/composition.overlay.html + - "**/*.html" + - "**/*.css" +--- + +Surface is the only way an element gets a background, border, radius, or +shadow. If a container needs visual treatment, it first needs a surface role. + +The vocabulary is closed: role, padding, radius, border, and elevation. These +axes create enough range for page flow, cards, popovers, muted blocks, +accents, and dark moments without inventing one-off boxes. + +The default surface is flat: no border, no shadow. We do not outline +everything to prove layout exists. + +The elevation tiers are exactly three: card, popover, modal. Elevation +implies hierarchy — a card sits in the document flow, a popover floats above +the flow, a modal interrupts the task. Pick the tier that matches the +interaction; never write a custom shadow because the composition feels flat, +and never pair a low tier with an interrupting role or give a routine card +modal gravity. Component shadows belong to the primitives that own them; +never borrow them for layout. + +Borders are structural, especially for inputs and overlays. They are not +decorative frames around ordinary text — reject borders as decoration, +especially around assistant text, and use space, tone, and type hierarchy +instead. + +The overlay reference shows the interrupting end of the system: background +scrim, modal radius, modal shadow, compact header, clear footer. Copy its +hierarchy when a user must stop and decide. diff --git a/packages/vessel-light/.ghost/index.md b/packages/vessel-light/.ghost/index.md new file mode 100644 index 00000000..7b01cbc2 --- /dev/null +++ b/packages/vessel-light/.ghost/index.md @@ -0,0 +1,37 @@ +--- +description: "Always read first: Vessel's design language as a raw HTML/CSS steering packet for agents." +materials: + - materials/tokens.css +--- + +vessel-light is Vessel without the React package: the design language as a +repo-local steering packet for agents writing raw HTML and CSS. + +The contract is narrow. Style only with the tokens. Compose only with the +closed sets the grammar enumerates. Imitate the refs when the task matches +them — they are worked examples to copy from, not a framework to import. + +The corpus carries three trust tiers. Grammar and the median floor are law: +safe to consume verbatim, unchanged by any adaptation. Signature nodes are +dials awaiting your values: each states Vessel's current answer and stands +until you replace it. Registers are conditions: editorial, email, and +data-density each name the situation where parts of the default contract +invert. Gather against the actual task and pull only the nodes whose +conditions apply, because truths from the wrong register are poison, not +context. + +The reset raises the floor; only the fingerprint escapes the median. + +Monochrome comes first. Status color is functional language for danger, +success, warning, and information — never atmosphere. Atmosphere exists, but +it is catered: a five-hue expression palette whose volume rises with the +register, from charts-only in product to two hues in editorial, and never on +controls. + +The fingerprint protects the taste that disappears when an agent freehands: +quiet surfaces, deliberate spacing, one obvious action, and restrained +motion. + +When a requested UI is not covered by a ref, compose from the grammar. Do +not invent a new styling system. If the result needs a new pattern, the new +pattern should come from observed drift, not anticipation. diff --git a/packages/vessel-light/.ghost/manifest.yml b/packages/vessel-light/.ghost/manifest.yml new file mode 100644 index 00000000..3cd8d93f --- /dev/null +++ b/packages/vessel-light/.ghost/manifest.yml @@ -0,0 +1,2 @@ +schema: ghost.fingerprint-package/v1 +id: vessel-light diff --git a/packages/vessel/src/fonts/HKGrotesk-Black.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Black.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-Black.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Black.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-Bold.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Bold.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-Bold.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Bold.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-ExtraBold.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-ExtraBold.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-ExtraBold.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-ExtraBold.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-Light.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Light.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-Light.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Light.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-Medium.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Medium.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-Medium.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Medium.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-Regular.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Regular.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-Regular.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-Regular.woff2 diff --git a/packages/vessel/src/fonts/HKGrotesk-SemiBold.woff2 b/packages/vessel-light/.ghost/materials/fonts/HKGrotesk-SemiBold.woff2 similarity index 100% rename from packages/vessel/src/fonts/HKGrotesk-SemiBold.woff2 rename to packages/vessel-light/.ghost/materials/fonts/HKGrotesk-SemiBold.woff2 diff --git a/packages/vessel-light/.ghost/materials/primitives.css b/packages/vessel-light/.ghost/materials/primitives.css new file mode 100644 index 00000000..999159b4 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/primitives.css @@ -0,0 +1,388 @@ +*, +*::before, +*::after { + box-sizing: border-box; +} + +body { + background: var(--background); + color: var(--foreground); + font-family: var(--font-sans); +} + +/* surface — defaults: background/foreground, radius-md, no padding, no border, no elevation. */ +.surface { + background: var(--background); + color: var(--foreground); + border: 0; + border-radius: var(--radius-md); + box-shadow: none; +} +.surface--card { + background: var(--card); + color: var(--card-foreground); +} +.surface--popover { + background: var(--popover); + color: var(--popover-foreground); +} +.surface--muted { + background: var(--muted); + color: var(--foreground); +} +.surface--accent { + background: var(--accent); + color: var(--accent-foreground); +} +.surface--dark { + background: var(--surface-dark); + color: var(--surface-dark-text); + border-color: var(--surface-dark-border); +} +.surface--pad-xs { + padding: 8px; +} +.surface--pad-sm { + padding: 12px; +} +.surface--pad-md { + padding: 16px; +} +.surface--pad-lg { + padding: 24px; +} +.surface--radius-sm { + border-radius: var(--radius-sm); +} +.surface--radius-md { + border-radius: var(--radius-md); +} +.surface--radius-lg { + border-radius: var(--radius-lg); +} +.surface--radius-surface { + border-radius: var(--radius-surface); +} +.surface--radius-control { + border-radius: var(--radius-control); +} +.surface--border { + border: 1px solid var(--border); +} +.surface--border-input { + border: 1px solid var(--input); +} +.surface--elevation-card { + box-shadow: var(--shadow-card); +} +.surface--elevation-popover { + box-shadow: var(--shadow-popover); +} +.surface--elevation-modal { + box-shadow: var(--shadow-modal); +} + +/* stack — defaults: column, gap-md(16px), stretch. Write class="stack" and you get vessel rhythm. */ +.stack { + display: flex; + flex-direction: column; + gap: var(--gap-md); + align-items: stretch; + justify-content: flex-start; + flex-wrap: nowrap; +} +.stack--row { + flex-direction: row; +} +.stack--gap-none { + gap: 0; +} +.stack--gap-xs { + gap: var(--gap-xs); +} +.stack--gap-sm { + gap: var(--gap-sm); +} +.stack--gap-md { + gap: var(--gap-md); +} +.stack--gap-lg { + gap: var(--gap-lg); +} +.stack--gap-xl { + gap: var(--gap-xl); +} +.stack--align-start { + align-items: flex-start; +} +.stack--align-center { + align-items: center; +} +.stack--align-end { + align-items: flex-end; +} +.stack--align-baseline { + align-items: baseline; +} +.stack--justify-center { + justify-content: center; +} +.stack--justify-end { + justify-content: flex-end; +} +.stack--justify-between { + justify-content: space-between; +} +.stack--wrap { + flex-wrap: wrap; +} + +/* text — defaults: body, 14px, relaxed, foreground. */ +.text { + margin: 0; + color: var(--foreground); + font-family: var(--font-sans); + font-size: var(--text-body-size); + line-height: 1.625; +} +.text--display { + font-size: var(--heading-display-font-size); + font-weight: 400; + line-height: 1; + letter-spacing: var(--heading-display-letter-spacing); +} +.text--headline { + font-size: var(--text-headline-size); + font-weight: 400; + line-height: 1.25; + letter-spacing: -0.025em; +} +.text--title { + font-size: var(--text-title-size); + font-weight: 600; + line-height: 1; + letter-spacing: -0.025em; +} +.text--label { + font-size: var(--label-font-size); + font-weight: var(--label-font-weight); + line-height: var(--label-line-height); + letter-spacing: var(--label-letter-spacing); +} +.text--mono { + font-family: var(--font-mono); + font-size: var(--text-body-size); + line-height: 1.5; +} +.text--muted { + color: var(--muted-foreground); +} +.text--inverse { + color: var(--primary-foreground); +} +.text--success { + color: var(--success); +} +.text--warning { + color: var(--warning); +} +.text--info { + color: var(--info); +} +.text--destructive { + color: var(--destructive); +} + +/* button — defaults: primary pill, 36px high, centered, fast focusable action. */ +.button { + display: inline-flex; + align-items: center; + justify-content: center; + gap: var(--gap-sm); + min-width: 0; + height: var(--control-height); + padding: 8px 24px; + border: 1px solid transparent; + border-radius: var(--radius-control); + background: var(--primary); + color: var(--primary-foreground); + box-shadow: none; + font: inherit; + font-size: var(--text-body-size); + line-height: 1; + text-decoration: none; + white-space: nowrap; + cursor: pointer; + transition: + background var(--duration-fast) var(--ease-spring), + border-color var(--duration-fast) var(--ease-spring), + box-shadow var(--duration-fast) var(--ease-spring), + opacity var(--duration-fast) var(--ease-spring); +} +.button:hover { + box-shadow: var(--shadow-btn); +} +.button:focus-visible { + outline: none; + border-color: var(--ring); + box-shadow: 0 0 0 1px var(--ring); +} +.button:disabled, +.button[aria-disabled="true"] { + pointer-events: none; + opacity: 0.5; +} +.button--primary { + background: var(--primary); + color: var(--primary-foreground); +} +.button--secondary { + background: var(--muted); + color: var(--foreground); +} +.button--ghost { + background: transparent; + color: var(--foreground); +} +.button--ghost:hover, +.button--secondary:hover, +.button--outline:hover { + background: var(--muted); +} +.button--destructive { + background: var(--destructive); + color: var(--destructive-foreground); +} +.button--outline { + background: var(--background); + color: var(--foreground); + border-color: var(--input); +} +.button--link { + padding-inline: 4px; + background: transparent; + color: var(--foreground); + text-decoration: underline; + text-underline-offset: 3px; + transition: color var(--duration-fast) var(--ease-spring); +} +.button--link:hover { + background: transparent; + color: var(--muted-foreground); + box-shadow: none; +} +.button--size-xs { + height: 24px; + padding: 4px 8px; + gap: var(--gap-xs); + font-size: 12px; +} +.button--size-sm { + height: 32px; + padding: 6px 16px; + gap: 6px; +} +.button--size-lg { + height: 40px; + padding: 8px 32px; +} +.button--icon, +.button--icon-sm, +.button--icon-xs, +.button--icon-lg { + padding: 0; +} +.button--icon { + width: var(--control-height); + height: var(--control-height); +} +.button--icon-xs { + width: 24px; + height: 24px; +} +.button--icon-sm { + width: 32px; + height: 32px; +} +.button--icon-lg { + width: 40px; + height: 40px; +} + +/* field — defaults: label plus pill input with input border, muted placeholder, ring focus, compact error. */ +.field { + display: flex; + flex-direction: column; + gap: var(--gap-sm); +} +.label { + display: flex; + gap: var(--gap-sm); + color: var(--foreground); + font-size: var(--text-body-size); + line-height: 1; + user-select: none; +} +.input { + display: flex; + width: 100%; + min-width: 0; + height: var(--control-height); + padding: 4px 12px; + border: 1px solid var(--input); + border-radius: var(--radius-input); + background: transparent; + color: var(--foreground); + font: inherit; + font-size: 16px; + outline: none; + transition: + border-color var(--duration-fast) var(--ease-spring), + box-shadow var(--duration-fast) var(--ease-spring); +} +.input::placeholder { + color: var(--muted-foreground); +} +.input:hover { + border-color: var(--border); +} +.input:focus, +.input:focus-visible { + border-color: var(--ring); + box-shadow: 0 0 0 1px var(--ring); +} +.input[aria-invalid="true"] { + border-color: var(--destructive); + box-shadow: 0 0 0 1px var(--destructive); +} +.input--textarea { + min-height: 64px; + height: auto; + padding: 12px; + resize: vertical; + border-radius: var(--radius-md); + line-height: 1.5; +} +.input--select { + align-items: center; + justify-content: space-between; + cursor: default; +} +.input--checkbox { + width: 16px; + height: 16px; + padding: 0; + flex: 0 0 16px; + border-radius: var(--radius-sm); + accent-color: var(--primary); +} +.field-error { + color: var(--destructive); + font-size: 12px; + line-height: 1.4; +} + +/* scrim — default overlay wash uses the tokenized alpha surface. */ +.scrim { + background: var(--dark-40); +} diff --git a/packages/vessel-light/.ghost/materials/ref/composition.conversation.html b/packages/vessel-light/.ghost/materials/ref/composition.conversation.html new file mode 100644 index 00000000..7d7e9a5b --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/composition.conversation.html @@ -0,0 +1,63 @@ + + + + + + + Vessel Light — Conversation + + + + + +
+
+
+

Can you make the billing page feel calmer without losing the upgrade prompt?

+
+ +
+

Yes. I would reduce the number of competing panels, keep the plan state explicit, and move the upgrade prompt into a single bordered decision area.

+

The page should feel like account infrastructure first, promotion second.

+
+ +
+ + Inspecting tokens + Completed + + 
read packages/vessel/src/components/billing-card.tsx
+
+ +
+

Use one card surface for the current plan, muted rows for secondary facts, and a primary action only where the user can commit. Keep helper copy short and concrete.

+
+ +
+

Show me the smallest set of changes.

+
+
+ +
+ +
+
+ + +
+ +
+
+
+ + diff --git a/packages/vessel-light/.ghost/materials/ref/composition.editorial.html b/packages/vessel-light/.ghost/materials/ref/composition.editorial.html new file mode 100644 index 00000000..ba68d064 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/composition.editorial.html @@ -0,0 +1,69 @@ + + + + + + + Vessel Light — Editorial Landing + + + + + +
+
+

Independent review layer

+

Make every generated screen answer to the same brand memory.

+

Vessel keeps interface materials plain, inspectable, and close to the repo so agents can preserve the feel behind the product, not just the surface.

+ Review materials +
+ +
+
+

A small set of primitives carries the whole system.

+
+
+

Density

+

Compact by default

+

Controls stay close to their labels. Surfaces reserve padding for decisions, not decoration.

+
+
+

Restraint

+

Monochrome first

+

Contrast, rhythm, and type scale do the work before color is asked to explain status.

+
+
+

Trust

+

Every action is explicit

+

Buttons name the commitment. Secondary copy clarifies consequence without trying to persuade.

+
+
+
+
+ +
+
The interface should feel authored even when an agent produced the first pass.
+
+
+ + diff --git a/packages/vessel-light/.ghost/materials/ref/composition.form.html b/packages/vessel-light/.ghost/materials/ref/composition.form.html new file mode 100644 index 00000000..87996e81 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/composition.form.html @@ -0,0 +1,89 @@ + + + + + + + Vessel Light — Settings Form + + + + + +
+
+
+

Workspace settings

+

Tune the defaults your team sees when Vessel drafts interface work.

+
+ +
+ +
+
+
+

Profile

+

These values appear on receipts, invites, and review packets.

+
+ +
+
+ + +
+ +
+ + +

Use a verified sender before inviting collaborators.

+
+ +
+ Default density +
+ Comfortable + +
+
+ + +
+ +
+ API access +
+ vsl_live_4f8a…03c2 + +
+

Regenerating invalidates the key currently in use by integrations.

+
+ +
+
+ Delete this workspace + Removes every fingerprint and review record immediately. +
+ +
+ + +
+
+
+ + diff --git a/packages/vessel-light/.ghost/materials/ref/composition.overlay.html b/packages/vessel-light/.ghost/materials/ref/composition.overlay.html new file mode 100644 index 00000000..f5ed74e7 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/composition.overlay.html @@ -0,0 +1,60 @@ + + + + + + + Vessel Light — Overlay + + + + + +
+
+

Fingerprint

+ +
+

Three nodes changed since the last publish. Review the diff, then publish from the actions menu.

+ + + +
+ + + + + diff --git a/packages/vessel-light/.ghost/materials/ref/composition.table.html b/packages/vessel-light/.ghost/materials/ref/composition.table.html new file mode 100644 index 00000000..1571ba6f --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/composition.table.html @@ -0,0 +1,104 @@ + + + + + + + Vessel Light — Data Table + + + + + +
+
+
+

Billing console

+

Recent payments

+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CustomerPlanInvoiceAmountStatus
+
+ Northstar Studio + design@northstar.example +
+ 		TeamVS-1048$96.00Paid
+
+ Common Room + ops@common.example +
+ 		BusinessVS-1047$240.00Paid
+
+ Field Notes Lab + admin@fieldnotes.example +
+ 		TeamVS-1046$96.00Paid
+
+ Atlas Review + finance@atlas.example +
+ 		StarterVS-1045$24.00Paid
+
+
+ + diff --git a/packages/vessel-light/.ghost/materials/ref/email.html b/packages/vessel-light/.ghost/materials/ref/email.html new file mode 100644 index 00000000..f4886438 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/ref/email.html @@ -0,0 +1,80 @@ + + + + + + + Payment receipt + + + + + + + +
+ + + + + + + + +
Vessel
+ + + + + + + + + + + + + + + + +
Payment receipt
Your payment for the Team plan has been received. A copy of this receipt is available in billing settings.
+ + + + + + + + + + + + + +
Receipt numberVS-1048
Billing dateJuly 7, 2026
Amount paid$96.00
+
+ + + + + +
+ View receipt +
+
If you have billing questions, reply to this email and include the receipt number.
+
+
+ + diff --git a/packages/vessel-light/.ghost/materials/tokens.css b/packages/vessel-light/.ghost/materials/tokens.css new file mode 100644 index 00000000..3f389f08 --- /dev/null +++ b/packages/vessel-light/.ghost/materials/tokens.css @@ -0,0 +1,242 @@ +/* Vessel Light materials contract: style only with these custom properties; never hardcode colors, shadows, or timing. */ + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-Light.woff2") format("woff2"); + font-weight: 300; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-Regular.woff2") format("woff2"); + font-weight: 400; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-Medium.woff2") format("woff2"); + font-weight: 500; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-SemiBold.woff2") format("woff2"); + font-weight: 600; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-Bold.woff2") format("woff2"); + font-weight: 700; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-ExtraBold.woff2") format("woff2"); + font-weight: 800; + font-style: normal; + font-display: swap; +} + +@font-face { + font-family: "HK Grotesk"; + src: url("fonts/HKGrotesk-Black.woff2") format("woff2"); + font-weight: 900; + font-style: normal; + font-display: swap; +} + +:root { + --font-sans: "HK Grotesk", system-ui, sans-serif; + --font-mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace; + + --color-white: #ffffff; + --color-black: #000000; + + --color-gray-50: #f5f5f5; + --color-gray-100: #f0f0f0; + --color-gray-200: #e8e8e8; + --color-gray-300: #e5e5e5; + --color-gray-400: #cccccc; + --color-gray-500: #999999; + --color-gray-600: #666666; + --color-gray-700: #333333; + --color-gray-800: #232323; + --color-gray-900: #1a1a1a; + + --color-red-100: #ff6b6b; + --color-red-200: #f94b4b; + --color-blue-100: #7cacff; + --color-blue-200: #5c98f9; + --color-green-100: #a3d795; + --color-green-200: #91cb80; + --color-yellow-100: #ffd966; + --color-yellow-200: #fbcd44; + + --radius: 20px; + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-surface: 20px; + --radius-control: 999px; + --radius-input: 999px; + + --gap-xs: 4px; + --gap-sm: 8px; + --gap-md: 16px; + --gap-lg: 24px; + --gap-xl: 32px; + + --text-body-size: 14px; + --text-title-size: 18px; + --text-headline-size: 24px; + + --control-height: 36px; + + --background: var(--color-white); + --foreground: var(--color-gray-900); + --card: var(--background); + --card-foreground: var(--foreground); + --popover: var(--background); + --popover-foreground: var(--foreground); + --primary: var(--color-gray-900); + --primary-foreground: var(--color-white); + --secondary: var(--color-gray-100); + --secondary-foreground: var(--foreground); + --muted: var(--color-gray-100); + --muted-foreground: var(--color-gray-500); + --accent: var(--color-gray-100); + --accent-foreground: var(--foreground); + --destructive: var(--color-red-200); + --destructive-foreground: var(--color-white); + --success: var(--color-green-200); + --success-foreground: var(--color-gray-900); + --warning: var(--color-yellow-200); + --warning-foreground: var(--color-gray-900); + --info: var(--color-blue-200); + --info-foreground: var(--color-gray-900); + --border: var(--color-gray-200); + --input: var(--color-gray-300); + --ring: var(--color-gray-400); + + --dark-10: rgba(26, 26, 26, 0.1); + --dark-40: rgba(26, 26, 26, 0.4); + --dark-04: rgba(26, 26, 26, 0.04); + + --shadow-mini: 0 2px 8px rgba(76, 76, 76, 0.15); + --shadow-mini-inset: 0 1px 4px rgba(76, 76, 76, 0.1) inset; + --shadow-btn: 0 2px 8px rgba(76, 76, 76, 0.15); + --shadow-card: 0 2px 8px rgba(76, 76, 76, 0.15); + --shadow-elevated: 0 3px 12px rgba(76, 76, 76, 0.22); + --shadow-popover: 0 8px 30px rgba(0, 0, 0, 0.12); + --shadow-modal: 0 20px 60px rgba(0, 0, 0, 0.2); + --shadow-kbd: 0 2px 8px rgba(76, 76, 76, 0.15); + + --surface-dark: #0a0a0a; + --surface-dark-text: #f5f5f5; + --surface-dark-muted: rgba(255, 255, 255, 0.5); + --surface-dark-border: rgba(255, 255, 255, 0.08); + + --heading-display-font-size: clamp(64px, 8vw, 96px); + --heading-display-line-height: 0.88; + --heading-display-letter-spacing: -0.05em; + --heading-display-font-weight: 900; + --heading-section-font-size: clamp(44px, 5vw, 64px); + --heading-section-line-height: 0.95; + --heading-section-letter-spacing: -0.035em; + --heading-section-font-weight: 700; + --heading-sub-font-size: clamp(28px, 3vw, 40px); + --heading-sub-line-height: 1; + --heading-sub-letter-spacing: -0.02em; + --heading-sub-font-weight: 700; + --heading-card-font-size: clamp(20px, 2vw, 28px); + --heading-card-line-height: 1.1; + --heading-card-letter-spacing: -0.01em; + --heading-card-font-weight: 600; + + --display-size: clamp(3rem, 12vw, 12rem); + --display-line-height: 0.85; + --display-letter-spacing: -0.05em; + --body-reading-size: clamp(1rem, 1.3vw, 1.25rem); + --body-reading-line-height: 1.65; + --body-reading-letter-spacing: -0.01em; + --label-font-size: 11px; + --label-letter-spacing: 0.12em; + --label-font-weight: 600; + --label-line-height: 1.2; + --pullquote-size: clamp(1.5rem, 3vw, 2.5rem); + --pullquote-line-height: 1.3; + --pullquote-weight: 300; + --pullquote-letter-spacing: -0.02em; + + --page-container-max-width: 1440px; + --page-container-side-gutter: 20px; + --section-padding-vertical: 100px; + --section-heading-margin-bottom: 75px; + + --ease-spring: cubic-bezier(0.33, 1, 0.68, 1); + --duration-fast: 0.15s; + --duration-normal: 0.2s; + --duration-slow: 0.4s; + + /* Expression palette — the brand's five hues, volume-gated by register. + Product: charts only. Data-density: one hue family for status. + Email: exactly one expressive moment. Editorial: atmosphere, max two + hues per page. Never on interactive controls in any register. */ + --expression-1: #f6b44a; /* amber */ + --expression-2: #7585ff; /* periwinkle */ + --expression-3: #d76a6a; /* clay */ + --expression-4: #d185e0; /* orchid */ + --expression-5: #91cb80; /* sage */ +} + +.dark { + --background: var(--color-black); + --foreground: var(--color-white); + --card: var(--color-gray-800); + --card-foreground: var(--foreground); + --popover: var(--color-gray-800); + --popover-foreground: var(--foreground); + --primary: var(--color-white); + --primary-foreground: var(--color-black); + --secondary: var(--color-gray-700); + --secondary-foreground: var(--foreground); + --muted: var(--color-gray-800); + --muted-foreground: var(--color-gray-500); + --accent: var(--color-gray-800); + --accent-foreground: var(--foreground); + --destructive: var(--color-red-100); + --destructive-foreground: var(--color-black); + --success: var(--color-green-100); + --success-foreground: var(--color-black); + --warning: var(--color-yellow-100); + --warning-foreground: var(--color-black); + --info: var(--color-blue-100); + --info-foreground: var(--color-black); + --border: var(--color-gray-700); + --input: var(--color-gray-700); + --ring: var(--color-gray-400); + + --dark-10: rgba(242, 242, 242, 0.1); + --dark-40: rgba(242, 242, 242, 0.4); + --dark-04: rgba(242, 242, 242, 0.04); + + --shadow-mini: 0 2px 8px rgba(0, 0, 0, 0.4); + --shadow-mini-inset: 0 1px 4px rgba(0, 0, 0, 0.5) inset; + --shadow-btn: 0 2px 8px rgba(0, 0, 0, 0.3); + --shadow-card: 0 2px 8px rgba(0, 0, 0, 0.4); + --shadow-elevated: 0 3px 12px rgba(0, 0, 0, 0.5); + --shadow-popover: 0 8px 30px rgba(0, 0, 0, 0.4); + --shadow-modal: 0 20px 60px rgba(0, 0, 0, 0.6); + --shadow-kbd: 0 2px 8px rgba(0, 0, 0, 0.4); +} diff --git a/packages/vessel-light/.ghost/register.data-density.md b/packages/vessel-light/.ghost/register.data-density.md new file mode 100644 index 00000000..85adfcaf --- /dev/null +++ b/packages/vessel-light/.ghost/register.data-density.md @@ -0,0 +1,39 @@ +--- +description: "Data-dense consoles — gather for tables, dashboards, logs, and monitoring; compresses product rhythm to the two smallest gap steps, mono numerals, muted-first." +materials: + - materials/ref/composition.table.html + - materials/primitives.css +--- + +Condition: this node applies to data-dense surfaces — tables, dashboards, +transaction logs, and admin consoles. + +Data density inverts the settings-page rhythm. Operators need scan speed +before they need breathing room. + +Use the two smallest gap steps (`--gap-xs`, `--gap-sm`) where forms use the +medium step. Tight adjacency is meaning when rows compare against rows. + +Numerals are mono so columns align. A ragged amount column is a broken +instrument. + +Hierarchy is muted-first. Data is the default plane, labels are muted, and +emphasis is rare enough to stay useful. + +Data surfaces take the small radius. A surface-radius table cell is costume — +the signature radius (see signature.shape) belongs on cards, not cells. + +Status is a text label with at most one functional color family per view. + +Charts inside a console follow the product carve-out from the palette +signature: expression hues live inside the plot area only. Outside the plot, +the one-status-hue cap holds — chart color never leaks into rows, badges, or +headers. More status color turns monitoring into confetti. + +No zebra striping. Hairline borders carry rows with less noise and more +trust. + +Hover confirmation uses the fast duration. It should acknowledge targeting, +not animate the table. + +Generous whitespace is drift here. Wasted density makes operators scroll. diff --git a/packages/vessel-light/.ghost/register.editorial.md b/packages/vessel-light/.ghost/register.editorial.md new file mode 100644 index 00000000..27fd2602 --- /dev/null +++ b/packages/vessel-light/.ghost/register.editorial.md @@ -0,0 +1,41 @@ +--- +description: "Editorial and landing surfaces — gather for heroes, marketing pages, pull quotes, and full-bleed dark moments; inverts product type restraint." +materials: + - materials/ref/composition.editorial.html + - materials/tokens.css +--- + +Condition: this node applies to marketing, landing, and editorial surfaces — +never to product UI. + +Editorial surfaces invert the product type rules. Display scale is not an +indulgence here; it is the job. + +Use the `--heading-display-*` tokens — size, line height, and weight — when +the words are the composition. The display tier belongs to this register (see +signature.type for the scale's character). + +Section rhythm uses `--section-padding-vertical`, not a product stack +stretched until it looks important. + +The sanctioned dramatic moment is the full-bleed dark section. Use the +`--surface-dark-*` family and let contrast carry the scene. + +Pull quotes are visual punctuation for longform. They interrupt reading with +judgment, not decoration. + +The tracked uppercase label is the kicker grammar. It gives the page a hard +editorial edge before the headline lands. + +Monochrome remains the spine, but editorial is the loud end of the expression +ladder (see signature.palette): up to two expression hues per page as +atmosphere — a tinted dark section, a colored pull-quote accent, a duotone +image moment. Commit to few colors at scale; variety is where expression +collapses into decoration. + +There is one primary action per page, and it keeps the control radius from +signature.shape. Editorial confidence does not excuse competing calls to +action. + +A hero built at product scale reads as timid. Timidity is drift here, exactly +as spectacle is drift in product UI. diff --git a/packages/vessel-light/.ghost/register.email.md b/packages/vessel-light/.ghost/register.email.md new file mode 100644 index 00000000..3517f07e --- /dev/null +++ b/packages/vessel-light/.ghost/register.email.md @@ -0,0 +1,43 @@ +--- +description: "Transactional email — gather ONLY for email; inverts the token contract: inline styles, hardcoded hex transcribed from tokens.css, table layout, system fonts." +materials: + - materials/ref/email.html +--- + +Condition: this node applies only to email. In any other medium, everything +below is a violation. + +Email clients do not honor the web contract. Custom properties, external +stylesheets, flex layout, and webfonts are unreliable materials here. + +Email inverts the material contract deliberately. Transcribe token values by +hand instead of referencing tokens — every hex below is a transcription of +`materials/tokens.css`, which stays the single source; if a token changes, +re-transcribe. + +Use `#1a1a1a` (`--color-gray-900`, foreground) for text, `#999999` +(`--color-gray-500`, muted) for muted text, and `#e8e8e8` (`--color-gray-200`, +border) for borders when the email needs the Vessel palette. + +Keep the surface radius on cards and the control radius on buttons (see +signature.shape). The values survive even when the token names cannot travel. + +Build with table layout, a 600px wrapper, predictable cells, and bulletproof +buttons. Email fidelity is made from boring structure. + +HK Grotesk falls back to the system stack. The voice must survive without the +font file. + +The soul survives the body swap: monochrome spine, quiet factual copy, one +primary action, no celebration. + +Email gets exactly one expressive moment: a header band or the figure that +matters, in one expression hue, transcribed by hand like every value here — +amber is `#f6b44a` (`--expression-1`). One moment, one hue. A receipt is +allowed one degree of warmth; it is not allowed a palette. + +Do not add further color to compensate for email constraints. Constraint is +not permission to perform. + +Hardcoding hex here is fidelity, not drift. The check exemption is the +condition itself. diff --git a/packages/vessel-light/.ghost/signature.palette.md b/packages/vessel-light/.ghost/signature.palette.md new file mode 100644 index 00000000..1669e57e --- /dev/null +++ b/packages/vessel-light/.ghost/signature.palette.md @@ -0,0 +1,45 @@ +--- +description: "Gather whenever color beyond the base roles is in question, in any register. The palette dial: a monochrome spine plus a closed expression set whose volume rises with the register — this brand's current answer is gray plus five named hues." +materials: + - materials/tokens.css +--- + +This is Vessel's answer to palette — it stands until you replace it. + +The relationship is fixed: one brand at different volumes. A monochrome spine +is the default atmosphere in every medium — calm, legible, and resistant to +novelty — and a closed expression set supplies the atmosphere, with volume +set by register, never by taste. + +Vessel's current answer: the base palette is monochrome gray, and the +expression palette is five named hues and only these — amber +(`--expression-1`), periwinkle (`--expression-2`), clay (`--expression-3`), +orchid (`--expression-4`), sage (`--expression-5`). Expressive color outside +this set is not expression; it is another brand. An invented hue is not a +bolder Vessel; it is a different brand. + +The volume ladder, by register: + +Product UI: expression lives only in data visualization. A chart may use the +hues; the interface around it stays monochrome. + +Data-dense consoles: one hue family may mark status. Nothing atmospheric. +Charts inside a console keep the product carve-out — hues stay inside the +plot area and never leak into rows, badges, or headers. + +Email: exactly one expressive moment per message — a header band or the +figure that matters. One hue, quiet everywhere else. + +Editorial: expression is sanctioned atmosphere — a tinted dark section, a +colored pull-quote accent, a duotone moment. Never more than two hues per +page. Loudness comes from commitment to few colors at scale, not variety. + +Two constants that outrank the ladder: expression never touches what you +click — buttons, inputs, and links stay monochrome in every register — and +the status roles are not expression; they keep their meanings everywhere and +never moonlight as atmosphere. + +To adapt: edit the `--expression-*` values (and the gray ramp, if the spine +changes) in `materials/tokens.css` and restate this node's current answer. +The ladder — a quiet spine, a closed hue set, register-gated volume — is the +part worth keeping. diff --git a/packages/vessel-light/.ghost/signature.shape.md b/packages/vessel-light/.ghost/signature.shape.md new file mode 100644 index 00000000..1ccd7f0f --- /dev/null +++ b/packages/vessel-light/.ghost/signature.shape.md @@ -0,0 +1,27 @@ +--- +description: "Gather before setting any radius or corner treatment. The shape dial: controls take --radius-control, surfaces take --radius-surface — this brand's current answer is pills on controls and a 20px signature radius on surfaces." +materials: + - materials/tokens.css + - materials/primitives.css +--- + +This is Vessel's answer to shape — it stands until you replace it. + +The relationship is fixed: controls and surfaces carry different radius +roles, and the two never swap. Buttons and text inputs use +`--radius-control`; cards and other surfaces use `--radius-surface`. + +Vessel's current answer: controls are pills (`--radius-control: 999px`) and +surfaces take the 20px signature radius (`--radius-surface: 20px`). The pill +is Vessel's most visible control signature; a rectangular button is not a +variant, it is a different design system. Never give a button the surface +radius — a rectangular button is the fastest tell that the output is not +Vessel. + +20px is a considered position: rounder than default shadcn, but not bubbly. +Use the radius system as restraint, not as decoration. + +To adapt: edit the `--radius-control` and `--radius-surface` values in +`materials/tokens.css` and restate this node's current answer. The +role split — one radius for what you click, one for what contains — is the +part worth keeping. diff --git a/packages/vessel-light/.ghost/signature.temperature.md b/packages/vessel-light/.ghost/signature.temperature.md new file mode 100644 index 00000000..88e7c498 --- /dev/null +++ b/packages/vessel-light/.ghost/signature.temperature.md @@ -0,0 +1,37 @@ +--- +description: "Gather for any copy and for motion character. The temperature dial: how the brand sounds and how it moves — this brand's current answer is factual, quiet, never celebratory, with resolved spring motion." +materials: + - materials/tokens.css +--- + +This is Vessel's answer to temperature — it stands until you replace it. + +The relationship is fixed: voice and motion carry the same temperature. Copy +states what happened, what is possible, or what the user must decide — it +does not perform personality — and motion confirms rather than entertains. + +Vessel's current answer in words: product UI is the default register — +factual, quiet, sentence case, and free of applause. No exclamation marks; +celebration is not reassurance. Confirmations are quiet: "Changes saved", +never "Awesome!" The system should sound reliable, not excited by basic +competence. Errors name the problem and the fix in the place where the user +can act; a vague failure banner is evasion. Buttons name the act — "Delete +account", "Save changes", "Invite member", not "Do it" or "Let's go". +Destructive copy is direct because the risk is direct; softening the verb +makes the interface less honest. Vessel never celebrates at the user. Trust +comes from precision, restraint, and naming the truth plainly. + +Vessel's current answer in motion: three durations — `--duration-fast`, +`--duration-normal`, `--duration-slow` — and one ease, `--ease-spring`, which +should feel resolved without feeling elastic. Hover confirms with color and +background shifts at the fast duration, not levitation. + +Condition: editorial surfaces switch register. Short declarative confident +fragments are correct in heroes and section headings. That same editorial +confidence is wrong in a settings form — product copy serves the task before +it serves the voice. Email copy uses the product voice: factual, not +campaign-like. + +To adapt: rewrite this node's current answers in your brand's voice and edit +the duration and ease values in `materials/tokens.css`. The coupling — words +and motion sharing one temperature — is the part worth keeping. diff --git a/packages/vessel-light/.ghost/signature.type.md b/packages/vessel-light/.ghost/signature.type.md new file mode 100644 index 00000000..f5d59003 --- /dev/null +++ b/packages/vessel-light/.ghost/signature.type.md @@ -0,0 +1,36 @@ +--- +description: "Gather for any text, and for heroes, landing pages, and editorial moments. The type dial: one voice typeface plus a mono for machine detail, with an editorial heading scale kept separate from product text — this brand's current answer is HK Grotesk." +materials: + - materials/tokens.css + - materials/fonts/*.woff2 +--- + +This is Vessel's answer to type — it stands until you replace it. + +The relationship is fixed: one typeface is the voice of the interface, +everywhere, with a mono variant only for code, tool detail, and machine +output. The heading scale is editorial — its tokens exist for pages composed +outside the text variants — and product UI never mixes the two vocabularies +in one view. + +Vessel's current answer: HK Grotesk. vessel-light ships it as its embedded +voice — here, `tokens.css` is canonical. + +The heading scale is editorial: display, section, sub, and card each carry +their own rhythm (`--heading-display-*`, `--heading-section-*`, +`--heading-sub-*`, `--heading-card-*`). Display sizes use tight tracking and +sub-1 line heights because the words behave like composition, not paragraph +text. + +Labels are small, semibold, and tracked wide. They should feel precise, never +loud. Body text keeps the established reading sizes with relaxed line height; +do not shrink important prose until it becomes legal copy. + +Condition: display and section scale are for editorial or hero moments. +Product UI lives in headline, title, body, label, and mono. A modal title is +not a poster. When in doubt, make type quieter and layout clearer. + +To adapt: swap the font files, edit `--font-sans`, `--font-mono`, and the +heading tokens in `materials/tokens.css`, and restate this node's current +answer. The one-voice rule and the editorial/product vocabulary split are the +parts worth keeping. diff --git a/packages/vessel-light/README.md b/packages/vessel-light/README.md new file mode 100644 index 00000000..7df8c5dc --- /dev/null +++ b/packages/vessel-light/README.md @@ -0,0 +1,32 @@ +# vessel-light + +vessel-light is Vessel's design language as a locked-and-loaded `.ghost/` fingerprint for agents writing raw HTML/CSS. No package. No build. No dependencies. Copy the fingerprint into a repo, hand it to `ghost`, and make the agent style with the same tokens, primitives, and worked compositions Vessel uses. + +## Copy + +```bash +npx degit /packages/vessel-light/.ghost .ghost +``` + +Or copy it without degit: + +```bash +cp -R packages/vessel-light/.ghost .ghost +``` + +## Loop + +```bash +ghost gather +ghost pull [...] +# generate HTML/CSS +ghost review +``` + +## Structure + +The corpus is factored by rate of change under adaptation: `grammar.*` nodes are value-free decision logic that survives any adaptation; `signature.*` nodes are the identity dials (shape, palette, type, temperature), each stating Vessel's current answer; `register.*` nodes are named conditions that re-tune the contract; `anti-goal.median` is the model's measured defaults (prune lines your brand legitimately violates) and `anti-goal.tells` guards near-misses of Vessel's own signature. Every literal value lives in `materials/tokens.css`. + +## Curation + +Each ref carries an annotation header declaring what it is normative for and what is incidental; together the refs demonstrate every closed set the grammar enumerates (the emphasis ladder, text variants, tones, elevation tiers). Regenerate refs whenever tokens or signature nodes change — a stale exemplar steers harder than any prose. Promote a new ref when repeated work proves a gap; do not add coverage upfront because a surface might exist someday. diff --git a/packages/vessel-light/demo/.gitignore b/packages/vessel-light/demo/.gitignore new file mode 100644 index 00000000..e362c4c5 --- /dev/null +++ b/packages/vessel-light/demo/.gitignore @@ -0,0 +1,3 @@ +# Eval run artifacts are never committed; regenerate via steering-eval +# (prompt → generate → shoot → score → report) from asks.md + ballast.md. +out/ diff --git a/packages/vessel-light/demo/README.md b/packages/vessel-light/demo/README.md new file mode 100644 index 00000000..7f7280f3 --- /dev/null +++ b/packages/vessel-light/demo/README.md @@ -0,0 +1,88 @@ +# vessel-light steering demo + +An A/B/C comparison showing why `ghost gather` (just-in-time selection) beats +context-dumping — especially under production context pressure. + +## Arms + +| Arm | Context | Claim it tests | +| --- | --- | --- | +| **A — naked** | ballast + ask | baseline: median model output, no brand | +| **B — dump** | fingerprint dump (front) + ballast + ask | the naive integration: "paste the brand guide in the system prompt" | +| **C — gather** | ballast + gather menu + selected pulls (near the ask) + ask | Ghost's actual loop | + +Optional: **B-clean** (dump, no ballast) vs **B-loaded** isolates attention +dilution — same fingerprint, degrading as the window fills. + +## Why the corpus is built for this + +The fingerprint spans registers that are individually true and mutually +contradictory outside their conditions: + +- product ↔ editorial (quiet type vs. 96px display heroes) +- product ↔ email (token discipline vs. mandatory hardcoded hex) +- product ↔ data-density (16px rhythm vs. 4/8px compression) + +Dumped whole, the conditions blur and generations average across registers. +Gathered, the agent selects the register the task belongs to. + +## Asks (boundary-adjacent by design) + +1. "Build a billing settings page" — product ask; editorial scale and email + hardcoding are the poison. +2. "Build a pricing landing page" — editorial ask; product restraint poisons + *downward* (timid heroes). +3. "Build a payment-receipt email" — material-level failure mode; `var(--…)` + in email breaks in clients, mechanically checkable. + +## Protocol + +The demo is driven by `steering-eval` (`packages/steering-eval`), configured +by `eval.config.json`. `ballast.md` is fixed and identical across arms +(~77K words / ~100K tokens of realistic-shaped, irrelevant session context). +Never edit it between runs. + +Per cell (arm × ask), for k = 1..5 — gather runs strictly serialized: + +```bash +node ../../steering-eval/cli.mjs prompt --run +# hand the prompt file to a FRESH agent context; it writes run-.html +# (gather arm: the agent runs `ghost pull` itself mid-task — that IS the test) +node ../../steering-eval/cli.mjs finish +``` + +Then, zero LLM calls, rebuildable from `out/` alone: + +```bash +node ../../steering-eval/cli.mjs shoot && node ../../steering-eval/cli.mjs score && node ../../steering-eval/cli.mjs report +``` + +Fairness rules: B's dump goes up front (that is where the naive integration +puts it); C's pulls land after ballast, near the ask (that is how the CLI +is used mid-conversation). Same ballast, same asks, same model, same +temperature everywhere. Report distributions, never single wins. + +## Median score + +`median-score.mjs` counts measured unsteered-generation tells (from the +antimedian experiment: 300 generations, 3 models, no design context) in any +HTML artifact. Calibration on real data: + +- antimedian unsteered pages: mean **9.1/22** (gpt runs up to 15/22) +- vessel-light refs: mean **0.2/22** + +Score every arm's artifacts; report the mean per arm alongside `ghost review` +results. Known caveat: the font tells (`inter-font-default`, +`segoe-font-default`) legitimately fire on email artifacts, where the system +stack is mandated by `register.email` — discount them there. + +## Files + +- `eval.config.json` — the steering-eval configuration (arms, k, corpora) +- `ballast.md` — the fixed context ballast +- `asks.md` — the three asks, plus `expect:`/`poison:`/`discount:` harness + metadata (stripped before generation; minimal, defensible sets only) +- `run-arm.md` — per-arm assembly instructions for the driving agent +- `median-tells.json` — the distilled tell list with antimedian evidence counts +- `median-score.mjs` — the standalone scorer (steering-eval embeds the same + tell-matching) diff --git a/packages/vessel-light/demo/asks.md b/packages/vessel-light/demo/asks.md new file mode 100644 index 00000000..b600c5bd --- /dev/null +++ b/packages/vessel-light/demo/asks.md @@ -0,0 +1,33 @@ +# Demo asks (verbatim, same across arms) + +`expect:` / `poison:` / `discount:` lines are harness metadata — the asks +parser strips them; they never enter generation context. Sets are minimal +and defensible: contested nodes are listed nowhere and score as neutral. + +## Ask 1 — billing settings page + +Build a billing settings page for the Meridian workspace: plan summary, +payment method on file, billing email, invoice history, and a way to cancel +the subscription. Single HTML file. + +expect: grammar.hierarchy, grammar.rhythm, signature.shape +poison: register.email, register.editorial + +## Ask 2 — pricing landing page + +Build a pricing landing page for Meridian: hero, three plan tiers, one +customer quote, closing call to action. Single HTML file. + +expect: register.editorial, signature.palette, signature.type +poison: register.email +discount: unprompted-dark-theme + +## Ask 3 — payment-receipt email + +Build the payment-receipt email Meridian sends after a successful invoice +payment: amount, plan, date, card last-4, link to the invoice. Single HTML +file, must render in Outlook and Gmail. + +expect: register.email, signature.temperature +poison: register.editorial, register.data-density +discount: inter-font-default, segoe-font-default diff --git a/packages/vessel-light/demo/ballast.md b/packages/vessel-light/demo/ballast.md new file mode 100644 index 00000000..4c7d96af --- /dev/null +++ b/packages/vessel-light/demo/ballast.md @@ -0,0 +1,7325 @@ + + +# Session context (ballast — identical across demo arms) + +## System configuration +You are an assistant embedded in the Meridian operations platform. Follow +workspace policies. Tools available: search_orders, refund_payment, +get_customer, list_subscriptions, escalate_ticket, query_metrics. + +## User profile +- name: Dana Whitfield +- role: operations manager +- workspace: meridian-west +- plan: growth (seat count 14) +- locale: en-US, timezone America/Denver +- feature flags: new-billing-ui=false, bulk-export=true, ai-summaries=canary + +## Event log (recent) +``` +1751850119 login user=jordan session=s_46048 meta=v4 +1751850352 subscription_updated user=jordan session=s_23434 meta=v9 +1751850446 session_expired user=tom session=s_14165 meta=v1 +1751850546 invoice_paid user=marcus session=s_76237 meta=v1 +1751851125 invoice_paid user=jordan session=s_95181 meta=v9 +1751851559 invoice_paid user=tom session=s_87236 meta=v5 +1751851570 subscription_updated user=jordan session=s_65392 meta=v6 +1751851859 subscription_updated user=marcus session=s_54118 meta=v2 +1751851958 feature_flag_evaluated user=dana session=s_57052 meta=v6 +1751852233 login user=jordan session=s_70217 meta=v9 +1751852365 feature_flag_evaluated user=dana session=s_82357 meta=v5 +1751852740 session_expired user=marcus session=s_19116 meta=v1 +1751852978 card_declined user=dana session=s_40512 meta=v2 +1751853372 card_declined user=tom session=s_93320 meta=v6 +1751853543 support_ticket_opened user=priya session=s_37460 meta=v5 +1751853621 session_expired user=jordan session=s_32431 meta=v9 +1751853876 subscription_updated user=tom session=s_59735 meta=v5 +1751854451 invoice_paid user=jordan session=s_52504 meta=v1 +1751854690 login user=priya session=s_62581 meta=v5 +1751854762 invoice_paid user=elena session=s_51245 meta=v4 +1751855278 feature_flag_evaluated user=jordan session=s_70142 meta=v3 +1751855554 subscription_updated user=marcus session=s_83579 meta=v9 +1751855828 session_expired user=tom session=s_86484 meta=v7 +1751856203 invoice_paid user=marcus session=s_76784 meta=v8 +1751856301 login user=dana session=s_30033 meta=v3 +1751856738 session_expired user=dana session=s_60432 meta=v7 +1751857222 webhook_retried user=priya session=s_82512 meta=v1 +1751857344 webhook_retried user=priya session=s_94012 meta=v6 +1751857463 card_declined user=tom session=s_30730 meta=v8 +1751857471 card_declined user=elena session=s_33416 meta=v9 +1751857584 card_declined user=jordan session=s_76540 meta=v4 +1751857745 support_ticket_opened user=marcus session=s_80697 meta=v9 +1751857750 session_expired user=priya session=s_74042 meta=v1 + +## Cluster triage session 1 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +ledger-sync-9cbd1312e-tqbv6 1/1 Running 14 11m 10.42.7.71 ip-10-42-2-111.us-west-2.compute.internal +auth-gateway-ef932339a-q9bxm 1/1 Running 2 39h 10.42.7.154 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-c822533e1-swwrt 1/1 Running 14 34m 10.42.30.249 ip-10-42-16-244.us-west-2.compute.internal +export-scheduler-d793386be-7kqwa 1/1 Running 0 61h 10.42.13.17 ip-10-42-15-133.us-west-2.compute.internal +notify-dispatch-566f9c6bd-yuyza 1/1 Running 1 81h 10.42.13.88 ip-10-42-15-133.us-west-2.compute.internal +billing-worker-32a808411-g75m2 1/1 Running 14 13m 10.42.9.47 ip-10-42-2-111.us-west-2.compute.internal +search-indexer-149affae2-2k3ng 1/1 Running 1 20h 10.42.16.205 ip-10-42-2-206.us-west-2.compute.internal +notify-dispatch-124f29a84-kh6ut 1/1 Running 0 27h 10.42.1.36 ip-10-42-27-195.us-west-2.compute.internal +orders-api-814b949fb-gb9yz 1/1 Running 7 51m 10.42.3.81 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-b57e9df2a-cbu2n 1/1 Running 0 88h 10.42.23.226 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-969b6dc71-v7pvy 0/1 CrashLoopBackOff 14 7m 10.42.6.250 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-736431de0-2528d 1/1 Running 0 49h 10.42.16.124 ip-10-42-27-195.us-west-2.compute.internal +search-indexer-e865314ff-q96ku 1/1 Running 3 34m 10.42.0.221 ip-10-42-2-206.us-west-2.compute.internal +auth-gateway-a8c007769-ywzxb 1/1 Running 2 59h 10.42.13.132 ip-10-42-3-96.us-west-2.compute.internal +billing-worker-7d88314c0-fprfa 1/1 Running 0 94h 10.42.15.177 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-4ec0a5746-7ccy2 1/1 Running 7 31m 10.42.21.174 ip-10-42-13-183.us-west-2.compute.internal +export-scheduler-b5317e7f2-zrhmr 1/1 Running 2 17h 10.42.21.145 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-6d4f38345-jkpeg 1/1 Running 7 10m 10.42.24.174 ip-10-42-27-195.us-west-2.compute.internal +ledger-sync-50a4889c2-uq4ny 1/1 Running 2 85h 10.42.19.220 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-2c0d091ce-nwhv6 1/1 Running 2 48h 10.42.0.162 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-716468393-bu8w9 1/1 Running 14 31m 10.42.20.237 ip-10-42-27-195.us-west-2.compute.internal +ledger-sync-e77b2af84-u4svr 1/1 Running 0 3h 10.42.7.192 ip-10-42-27-195.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-13-183.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751963600 INFO [ingest-2] flush batch size=4096 dur_ms=1814 backlog=68027 +1751963607 INFO [ingest-0] gc pause exceeded budget pause_ms=1165 heap_mb=2608 +1751963619 WARN [ingest-5] retry attempt=1 for shard=0 after connection reset +1751963625 ERROR [ingest-0] retry attempt=2 for shard=16 after connection reset +1751963629 INFO [ingest-1] rebalance triggered generation=1 members=35 +1751963639 ERROR [ingest-7] slow consumer detected partition=7 lag=2783856 +1751963645 INFO [ingest-6] checkpoint written offset=3630494 epoch=7 +1751963653 WARN [ingest-1] rebalance triggered generation=7 members=46 +1751963661 INFO [ingest-3] dropping oversized record bytes=18611 topic=events.raw +1751963665 INFO [ingest-6] compaction pass complete segments=23 reclaimed_mb=1749 +1751963670 ERROR [ingest-5] checkpoint written offset=3024947 epoch=4 +1751963681 INFO [ingest-0] slow consumer detected partition=22 lag=2316025 +1751963687 WARN [ingest-4] gc pause exceeded budget pause_ms=1174 heap_mb=2901 +1751963696 INFO [ingest-1] gc pause exceeded budget pause_ms=2240 heap_mb=1281 +1751963702 ERROR [ingest-6] compaction pass complete segments=8 reclaimed_mb=935 +1751963708 INFO [ingest-1] flush batch size=4096 dur_ms=1917 backlog=46842 +1751963715 ERROR [ingest-2] rebalance triggered generation=8 members=6 +1751963724 ERROR [ingest-6] compaction pass complete segments=9 reclaimed_mb=2594 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2699 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 1: the flaky `checkout-flow` integration job + +**priya** (11:02): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (11:07): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (11:08): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (11:12): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (11:19): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (11:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (11:27): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (11:30): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (11:33): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (11:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (11:41): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (11:45): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16473702168 9m29s 1h +failure checkout-flow integration main push 16404755015 10m46s 3h +completed checkout-flow integration main push 16482860024 11m21s 5h +completed checkout-flow integration main push 16454422486 4m36s 7h +completed checkout-flow integration main push 16440568445 10m43s 9h +failure checkout-flow integration main push 16477278806 4m10s 11h +completed checkout-flow integration main push 16472620852 11m30s 13h +completed checkout-flow integration main push 16410140423 5m48s 15h +completed checkout-flow integration main push 16431497393 6m16s 17h +failure checkout-flow integration main push 16428427355 8m53s 19h +failure checkout-flow integration main push 16462817348 11m54s 21h +completed checkout-flow integration main push 16463746126 5m26s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 22 passed, 22 total +Tests: 1 failed, 180 passed, 222 total +Time: 248.681 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v1: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v1 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 1 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 1 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 1 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3307: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-400: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-589: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-641: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-756: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 1: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1751987204 INFO worker=w13 job=export:881160 chunk=9/12 flushed bytes=369680821 +1751987217 INFO worker=w07 heartbeat ok inflight=23 claimed_total=321 +1751987230 WARN scheduler queue depth 6966 exceeds soft limit 5000 +1751987240 INFO scheduler tick pending=1257 claimed=24 completed_last_min=359 +1751987256 INFO worker=w06 job=export:883502 rows=110583 bytes=192514739 dur_ms=6162 state=complete +1751987267 INFO worker=w03 job=export:883885 upload attempt=10 succeeded after retry +1751987283 INFO worker=w03 job=export:881383 chunk=4/12 flushed bytes=314131066 +1751987296 INFO worker=w05 heartbeat ok inflight=15 claimed_total=63 +1751987305 WARN scheduler queue depth 8041 exceeds soft limit 5000 +1751987318 INFO worker=w13 job=export:883316 chunk=2/12 flushed bytes=187375029 +1751987334 INFO worker=w15 job=export:882024 rows=897666 bytes=211066471 dur_ms=6605 state=complete +1751987347 INFO scheduler tick pending=2241 claimed=11 completed_last_min=146 +1751987357 DEBUG worker=w07 pool stats idle=10 active=9 waiting=0 +1751987369 INFO worker=w03 job=export:882305 upload attempt=1 succeeded after retry +1751987387 ERROR worker=w08 job=export:882263 upload attempt=2 failed: connection reset by peer, will retry +1751987397 ERROR worker=w09 job=export:881707 upload attempt=5 failed: connection reset by peer, will retry +1751987410 INFO scheduler tick pending=3881 claimed=4 completed_last_min=119 +1751987425 WARN scheduler queue depth 6657 exceeds soft limit 5000 +1751987435 ERROR worker=w11 job=export:883842 upload attempt=11 failed: connection reset by peer, will retry +1751987451 INFO scheduler tick pending=7943 claimed=8 completed_last_min=137 +1751987462 WARN scheduler queue depth 3486 exceeds soft limit 5000 +1751987476 DEBUG worker=w16 pool stats idle=25 active=12 waiting=0 +1751987490 INFO scheduler tick pending=4921 claimed=4 completed_last_min=360 +1751987502 WARN worker=w11 job=export:884340 lease renewal took 2046ms (budget 5000ms) +1751987513 WARN worker=w14 job=export:883508 lease renewal took 520ms (budget 5000ms) +1751987529 INFO worker=w01 job=export:884174 upload attempt=10 succeeded after retry +1751987539 INFO worker=w05 job=export:883707 state=claimed lease_ms=30000 +1751987551 INFO worker=w06 job=export:883863 upload attempt=11 succeeded after retry +1751987569 DEBUG worker=w05 pool stats idle=27 active=7 waiting=0 +1751987582 INFO worker=w16 job=export:883244 chunk=4/12 flushed bytes=245860778 +1751987594 DEBUG worker=w12 pool stats idle=12 active=8 waiting=0 +1751987604 INFO worker=w01 job=export:883281 chunk=3/12 flushed bytes=297737188 +1751987617 INFO scheduler tick pending=7038 claimed=31 completed_last_min=105 +1751987633 INFO worker=w04 job=export:884139 chunk=12/12 flushed bytes=358830211 +1751987646 ERROR worker=w13 job=export:881584 upload attempt=5 failed: connection reset by peer, will retry +1751987655 INFO worker=w03 job=export:883271 rows=419698 bytes=186289958 dur_ms=8061 state=complete +1751987672 INFO worker=w15 job=export:881014 rows=679426 bytes=34694202 dur_ms=4363 state=complete +1751987686 WARN worker=w03 job=export:883146 lease renewal took 2983ms (budget 5000ms) +1751987695 DEBUG worker=w06 pool stats idle=23 active=3 waiting=0 +1751987711 INFO worker=w01 heartbeat ok inflight=8 claimed_total=208 +1751987724 INFO worker=w04 job=export:881170 upload attempt=1 succeeded after retry +1751987734 INFO worker=w13 job=export:883230 state=claimed lease_ms=30000 +1751987747 INFO worker=w02 heartbeat ok inflight=31 claimed_total=371 +1751987763 INFO worker=w07 heartbeat ok inflight=5 claimed_total=328 +1751987774 WARN scheduler queue depth 2566 exceeds soft limit 5000 +1751987785 WARN worker=w01 job=export:883272 lease renewal took 3976ms (budget 5000ms) +1751987803 INFO worker=w15 job=export:883378 state=claimed lease_ms=30000 +1751987813 WARN worker=w16 job=export:883691 lease renewal took 4275ms (budget 5000ms) +1751987825 WARN scheduler queue depth 5490 exceeds soft limit 5000 +1751987841 INFO scheduler tick pending=7145 claimed=16 completed_last_min=167 +1751987853 WARN worker=w05 job=export:884323 lease renewal took 8194ms (budget 5000ms) +1751987867 INFO worker=w11 heartbeat ok inflight=7 claimed_total=316 +1751987881 INFO worker=w05 heartbeat ok inflight=18 claimed_total=180 +1751987890 INFO worker=w14 job=export:883718 rows=716401 bytes=358093459 dur_ms=5594 state=complete +1751987902 INFO worker=w01 job=export:883392 rows=398200 bytes=264200222 dur_ms=4340 state=complete +1751987919 DEBUG worker=w08 pool stats idle=10 active=1 waiting=0 +1751987932 WARN scheduler queue depth 4442 exceeds soft limit 5000 +1751987943 INFO worker=w02 heartbeat ok inflight=20 claimed_total=208 +1751987958 WARN scheduler queue depth 5276 exceeds soft limit 5000 +1751987968 WARN scheduler queue depth 2615 exceeds soft limit 5000 +1751987982 ERROR worker=w02 job=export:883687 upload attempt=6 failed: connection reset by peer, will retry +1751987995 INFO worker=w14 heartbeat ok inflight=22 claimed_total=83 +1751988006 WARN worker=w03 job=export:881785 lease renewal took 6480ms (budget 5000ms) +1751988019 INFO worker=w14 job=export:883354 state=claimed lease_ms=30000 +1751988034 DEBUG worker=w15 pool stats idle=20 active=9 waiting=0 +1751988050 INFO worker=w02 heartbeat ok inflight=1 claimed_total=61 +1751988062 WARN scheduler queue depth 4409 exceeds soft limit 5000 +1751988076 INFO worker=w10 job=export:884217 upload attempt=11 succeeded after retry +1751988085 DEBUG worker=w10 pool stats idle=6 active=1 waiting=0 +1751988099 WARN scheduler queue depth 5552 exceeds soft limit 5000 +1751988113 INFO worker=w14 job=export:881758 rows=897635 bytes=81509284 dur_ms=5965 state=complete +1751988127 INFO scheduler tick pending=3161 claimed=19 completed_last_min=349 +1751988136 INFO worker=w06 job=export:884254 chunk=6/12 flushed bytes=1231083 +1751988149 DEBUG worker=w07 pool stats idle=5 active=8 waiting=0 +1751988163 WARN scheduler queue depth 7654 exceeds soft limit 5000 +1751988180 WARN scheduler queue depth 3956 exceeds soft limit 5000 +1751988193 INFO scheduler tick pending=4557 claimed=25 completed_last_min=398 +1751988206 INFO scheduler tick pending=1579 claimed=15 completed_last_min=182 +1751988217 INFO worker=w15 job=export:881557 chunk=8/12 flushed bytes=249014110 +1751988227 INFO worker=w12 job=export:884445 chunk=2/12 flushed bytes=268742786 +1751988244 DEBUG worker=w09 pool stats idle=29 active=9 waiting=0 +1751988255 INFO worker=w05 job=export:883205 chunk=10/12 flushed bytes=42705232 +1751988270 WARN worker=w04 job=export:882611 lease renewal took 2178ms (budget 5000ms) +1751988279 DEBUG worker=w01 pool stats idle=24 active=3 waiting=0 +1751988293 INFO worker=w11 heartbeat ok inflight=20 claimed_total=132 +1751988308 INFO worker=w12 job=export:883636 chunk=9/12 flushed bytes=77446108 +1751988318 WARN scheduler queue depth 5815 exceeds soft limit 5000 +1751988333 INFO scheduler tick pending=5698 claimed=10 completed_last_min=25 +1751988346 INFO worker=w11 heartbeat ok inflight=28 claimed_total=156 +1751988357 INFO scheduler tick pending=6039 claimed=20 completed_last_min=22 +``` + + +## Cluster triage session 2 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +webhook-relay-be7f4489e-4yghp 1/1 Running 0 31h 10.42.10.204 ip-10-42-1-169.us-west-2.compute.internal +orders-api-9cabfa87d-kxdpv 1/1 Running 0 38h 10.42.4.213 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-1ef96ea87-88vna 1/1 Running 0 40h 10.42.13.79 ip-10-42-2-206.us-west-2.compute.internal +webhook-relay-8979a87af-rj5xa 1/1 Running 0 64h 10.42.11.9 ip-10-42-2-206.us-west-2.compute.internal +billing-worker-f4d7794b4-gb8df 1/1 Running 0 3h 10.42.22.114 ip-10-42-13-183.us-west-2.compute.internal +search-indexer-62600fd5d-hx5rr 0/1 CrashLoopBackOff 7 39m 10.42.16.239 ip-10-42-13-183.us-west-2.compute.internal +billing-worker-c4aec1250-my6jx 0/1 CrashLoopBackOff 7 21m 10.42.8.160 ip-10-42-2-206.us-west-2.compute.internal +auth-gateway-b7c72cc99-g6j49 1/1 Running 0 22h 10.42.18.7 ip-10-42-13-183.us-west-2.compute.internal +orders-api-c80f8db46-fqbdf 0/1 CrashLoopBackOff 14 28m 10.42.13.225 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-3cbf474d8-3hqze 1/1 Running 0 74h 10.42.7.213 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-2631fae81-tmpb2 1/1 Running 0 23h 10.42.16.132 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-e939ac98e-h5msy 1/1 Running 3 48m 10.42.19.149 ip-10-42-2-111.us-west-2.compute.internal +auth-gateway-1268ee778-msq4n 0/1 CrashLoopBackOff 7 30m 10.42.2.176 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-66348c38e-utbwx 1/1 Running 0 6h 10.42.19.100 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-dae59acb7-nfr3u 1/1 Running 7 15m 10.42.7.63 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-0d2a517d3-xx6fu 1/1 Running 2 2h 10.42.16.158 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-a4286a57a-q6fs2 1/1 Running 1 37h 10.42.7.138 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-d84aa8a69-c72gy 1/1 Running 0 69h 10.42.22.241 ip-10-42-12-115.us-west-2.compute.internal +export-scheduler-8b0ed6d8c-afrgu 1/1 Running 2 43h 10.42.6.134 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-677263798-rpmxa 1/1 Running 3 49m 10.42.2.126 ip-10-42-12-115.us-west-2.compute.internal +orders-api-4071cff71-ajkaj 1/1 Running 2 90h 10.42.29.141 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-b783b92e2-pmxav 1/1 Running 0 16h 10.42.15.66 ip-10-42-3-96.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-27-195.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751967203 INFO [ingest-2] retry attempt=6 for shard=15 after connection reset +1751967210 INFO [ingest-5] compaction pass complete segments=47 reclaimed_mb=2051 +1751967218 INFO [ingest-7] dropping oversized record bytes=34929 topic=events.raw +1751967225 ERROR [ingest-6] slow consumer detected partition=19 lag=84702 +1751967232 WARN [ingest-5] gc pause exceeded budget pause_ms=670 heap_mb=1032 +1751967237 INFO [ingest-7] gc pause exceeded budget pause_ms=1247 heap_mb=3283 +1751967247 INFO [ingest-1] dropping oversized record bytes=18644 topic=events.raw +1751967252 INFO [ingest-1] gc pause exceeded budget pause_ms=2128 heap_mb=1962 +1751967257 ERROR [ingest-1] checkpoint written offset=760115 epoch=3 +1751967267 INFO [ingest-1] checkpoint written offset=1228993 epoch=6 +1751967271 INFO [ingest-2] compaction pass complete segments=27 reclaimed_mb=2219 +1751967278 INFO [ingest-4] retry attempt=2 for shard=31 after connection reset +1751967287 WARN [ingest-1] retry attempt=4 for shard=22 after connection reset +1751967295 ERROR [ingest-7] compaction pass complete segments=35 reclaimed_mb=621 +1751967299 WARN [ingest-7] checkpoint written offset=1356645 epoch=7 +1751967307 INFO [ingest-1] retry attempt=4 for shard=22 after connection reset +1751967315 INFO [ingest-0] checkpoint written offset=2164674 epoch=6 +1751967320 WARN [ingest-7] checkpoint written offset=3181652 epoch=6 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2762 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 2: the flaky `checkout-flow` integration job + +**priya** (12:00): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (12:06): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (12:10): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (12:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (12:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (12:20): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (12:25): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (12:28): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (12:35): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (12:37): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (12:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (12:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16468013539 7m11s 1h +failure checkout-flow integration main push 16409623468 5m33s 3h +completed checkout-flow integration main push 16476910568 10m49s 5h +completed checkout-flow integration main push 16458811803 11m26s 7h +completed checkout-flow integration main push 16483084748 8m28s 9h +failure checkout-flow integration main push 16466366469 9m27s 11h +completed checkout-flow integration main push 16493926565 10m30s 13h +completed checkout-flow integration main push 16483880870 4m46s 15h +failure checkout-flow integration main push 16454525305 10m33s 17h +completed checkout-flow integration main push 16431596669 4m50s 19h +completed checkout-flow integration main push 16414094639 4m17s 21h +completed checkout-flow integration main push 16483194551 8m47s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 27 passed, 28 total +Tests: 1 failed, 232 passed, 209 total +Time: 213.330 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v2: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v1 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 2 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 2 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 2 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3314: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-474: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-501: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-691: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-788: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 2: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1751994401 INFO worker=w11 job=export:882543 rows=34855 bytes=154485219 dur_ms=7487 state=complete +1751994413 INFO scheduler tick pending=4725 claimed=26 completed_last_min=15 +1751994427 INFO worker=w08 job=export:881765 state=claimed lease_ms=30000 +1751994442 INFO scheduler tick pending=3087 claimed=7 completed_last_min=249 +1751994456 WARN worker=w05 job=export:883315 lease renewal took 4845ms (budget 5000ms) +1751994470 INFO worker=w05 job=export:884171 chunk=3/12 flushed bytes=324391142 +1751994479 INFO worker=w01 job=export:882052 upload attempt=8 succeeded after retry +1751994496 ERROR worker=w01 job=export:883174 upload attempt=6 failed: connection reset by peer, will retry +1751994506 DEBUG worker=w15 pool stats idle=10 active=7 waiting=0 +1751994522 INFO worker=w10 job=export:884452 rows=478351 bytes=170583705 dur_ms=7028 state=complete +1751994535 INFO worker=w10 job=export:884655 state=claimed lease_ms=30000 +1751994544 DEBUG worker=w15 pool stats idle=22 active=7 waiting=0 +1751994561 INFO worker=w02 job=export:882788 state=claimed lease_ms=30000 +1751994571 WARN scheduler queue depth 1660 exceeds soft limit 5000 +1751994587 WARN scheduler queue depth 975 exceeds soft limit 5000 +1751994600 INFO scheduler tick pending=8175 claimed=9 completed_last_min=274 +1751994608 WARN worker=w06 job=export:882035 lease renewal took 9137ms (budget 5000ms) +1751994623 INFO worker=w03 job=export:882855 state=claimed lease_ms=30000 +1751994639 INFO worker=w04 heartbeat ok inflight=22 claimed_total=252 +1751994647 INFO worker=w11 job=export:883747 chunk=7/12 flushed bytes=216167094 +1751994664 WARN worker=w01 job=export:882799 lease renewal took 1539ms (budget 5000ms) +1751994676 INFO worker=w01 job=export:882559 upload attempt=7 succeeded after retry +1751994689 WARN scheduler queue depth 4255 exceeds soft limit 5000 +1751994700 DEBUG worker=w05 pool stats idle=2 active=4 waiting=0 +1751994714 WARN worker=w11 job=export:881583 lease renewal took 1695ms (budget 5000ms) +1751994727 INFO scheduler tick pending=510 claimed=4 completed_last_min=192 +1751994740 INFO scheduler tick pending=4810 claimed=19 completed_last_min=125 +1751994756 INFO worker=w01 job=export:883152 state=claimed lease_ms=30000 +1751994768 INFO worker=w10 job=export:881499 rows=734949 bytes=290495579 dur_ms=9406 state=complete +1751994782 INFO worker=w02 job=export:882071 state=claimed lease_ms=30000 +1751994793 INFO worker=w09 heartbeat ok inflight=17 claimed_total=263 +1751994807 INFO worker=w14 job=export:882148 state=claimed lease_ms=30000 +1751994818 INFO worker=w03 job=export:884093 upload attempt=11 succeeded after retry +1751994830 INFO worker=w16 heartbeat ok inflight=8 claimed_total=80 +1751994842 DEBUG worker=w07 pool stats idle=7 active=3 waiting=0 +1751994856 INFO worker=w09 job=export:883658 upload attempt=7 succeeded after retry +1751994869 INFO worker=w12 job=export:881039 chunk=9/12 flushed bytes=359860255 +1751994882 INFO worker=w11 job=export:883971 rows=578607 bytes=369627295 dur_ms=8851 state=complete +1751994899 INFO worker=w06 job=export:883748 chunk=2/12 flushed bytes=133506589 +1751994911 INFO worker=w16 job=export:883110 upload attempt=4 succeeded after retry +1751994924 WARN worker=w10 job=export:881243 lease renewal took 4223ms (budget 5000ms) +1751994936 INFO worker=w11 job=export:883760 upload attempt=10 succeeded after retry +1751994950 INFO worker=w11 job=export:881474 state=claimed lease_ms=30000 +1751994959 INFO worker=w10 job=export:884241 rows=404748 bytes=321543299 dur_ms=6683 state=complete +1751994975 INFO worker=w07 job=export:881374 upload attempt=8 succeeded after retry +1751994988 DEBUG worker=w16 pool stats idle=2 active=3 waiting=0 +1751995000 INFO worker=w08 heartbeat ok inflight=16 claimed_total=179 +1751995013 INFO scheduler tick pending=5199 claimed=9 completed_last_min=12 +1751995025 INFO worker=w10 job=export:882314 chunk=11/12 flushed bytes=192199504 +1751995040 INFO worker=w15 job=export:882961 upload attempt=2 succeeded after retry +1751995050 INFO worker=w08 job=export:884965 state=claimed lease_ms=30000 +1751995067 WARN worker=w05 job=export:883809 lease renewal took 2794ms (budget 5000ms) +1751995080 WARN worker=w06 job=export:883478 lease renewal took 1468ms (budget 5000ms) +1751995094 INFO worker=w02 job=export:882677 state=claimed lease_ms=30000 +1751995102 INFO worker=w12 job=export:881755 upload attempt=1 succeeded after retry +1751995116 INFO worker=w02 job=export:882903 chunk=4/12 flushed bytes=153086313 +1751995130 INFO worker=w05 job=export:884343 upload attempt=7 succeeded after retry +1751995144 INFO scheduler tick pending=2140 claimed=15 completed_last_min=330 +1751995154 INFO worker=w16 job=export:881093 rows=939990 bytes=334447977 dur_ms=4811 state=complete +1751995168 INFO worker=w04 job=export:881923 rows=228816 bytes=270935882 dur_ms=6359 state=complete +1751995181 INFO scheduler tick pending=2573 claimed=16 completed_last_min=374 +1751995194 WARN worker=w16 job=export:884301 lease renewal took 7463ms (budget 5000ms) +1751995211 INFO worker=w11 job=export:881504 upload attempt=7 succeeded after retry +1751995220 WARN worker=w01 job=export:884576 lease renewal took 3854ms (budget 5000ms) +1751995234 INFO worker=w02 job=export:883825 upload attempt=11 succeeded after retry +1751995250 WARN scheduler queue depth 3409 exceeds soft limit 5000 +1751995262 INFO worker=w03 job=export:881268 chunk=6/12 flushed bytes=164215116 +1751995273 INFO worker=w14 heartbeat ok inflight=3 claimed_total=80 +1751995286 INFO worker=w10 job=export:883660 rows=850947 bytes=306445893 dur_ms=5348 state=complete +1751995299 DEBUG worker=w07 pool stats idle=13 active=5 waiting=0 +1751995311 INFO scheduler tick pending=3689 claimed=17 completed_last_min=292 +1751995328 INFO worker=w14 heartbeat ok inflight=25 claimed_total=88 +1751995339 INFO worker=w06 job=export:884459 rows=497899 bytes=83199881 dur_ms=8458 state=complete +1751995354 INFO worker=w04 job=export:882326 chunk=7/12 flushed bytes=181327178 +1751995366 INFO worker=w16 job=export:884315 upload attempt=2 succeeded after retry +1751995380 DEBUG worker=w08 pool stats idle=27 active=1 waiting=0 +1751995391 INFO worker=w16 job=export:883327 state=claimed lease_ms=30000 +1751995405 WARN worker=w13 job=export:883172 lease renewal took 9330ms (budget 5000ms) +1751995419 INFO scheduler tick pending=8100 claimed=17 completed_last_min=66 +1751995430 WARN worker=w11 job=export:882794 lease renewal took 892ms (budget 5000ms) +1751995444 INFO scheduler tick pending=4152 claimed=10 completed_last_min=259 +1751995457 INFO worker=w11 heartbeat ok inflight=32 claimed_total=46 +1751995468 INFO scheduler tick pending=7633 claimed=5 completed_last_min=152 +1751995479 INFO worker=w01 heartbeat ok inflight=29 claimed_total=89 +1751995496 INFO scheduler tick pending=3353 claimed=30 completed_last_min=289 +1751995505 INFO worker=w11 heartbeat ok inflight=8 claimed_total=378 +1751995520 INFO worker=w07 job=export:882375 upload attempt=2 succeeded after retry +1751995533 INFO worker=w05 job=export:883700 chunk=10/12 flushed bytes=113908470 +1751995548 INFO worker=w01 job=export:882119 state=claimed lease_ms=30000 +1751995561 WARN scheduler queue depth 7337 exceeds soft limit 5000 +``` + + +## Cluster triage session 3 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +orders-api-b85d99a60-nac43 1/1 Running 0 3h 10.42.26.197 ip-10-42-16-244.us-west-2.compute.internal +notify-dispatch-19e77c4a5-b59s6 1/1 Running 0 51h 10.42.11.142 ip-10-42-15-133.us-west-2.compute.internal +auth-gateway-4eada4135-4mppu 1/1 Running 0 68h 10.42.0.50 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-efa2628cd-k2xev 1/1 Running 0 89h 10.42.30.188 ip-10-42-16-244.us-west-2.compute.internal +rate-limiter-83921aab8-4f7ru 1/1 Running 14 44m 10.42.22.94 ip-10-42-15-133.us-west-2.compute.internal +export-scheduler-235b7710b-f9s64 1/1 Running 0 38h 10.42.12.82 ip-10-42-2-111.us-west-2.compute.internal +export-scheduler-2afc9839e-r86p8 0/1 CrashLoopBackOff 14 21m 10.42.30.36 ip-10-42-2-111.us-west-2.compute.internal +orders-api-3cbe56662-jemds 1/1 Running 7 27m 10.42.26.141 ip-10-42-12-115.us-west-2.compute.internal +rate-limiter-7746d180e-qsj5k 1/1 Running 2 78h 10.42.28.176 ip-10-42-27-195.us-west-2.compute.internal +notify-dispatch-fd97a8a0d-4wnc4 0/1 CrashLoopBackOff 7 27m 10.42.22.110 ip-10-42-1-169.us-west-2.compute.internal +notify-dispatch-a90e6e8a8-ajcwy 1/1 Running 7 55m 10.42.24.99 ip-10-42-27-195.us-west-2.compute.internal +orders-api-2464e46f8-w3bye 1/1 Running 0 53h 10.42.5.161 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-d4099fb2f-adpnc 1/1 Running 3 10m 10.42.17.199 ip-10-42-2-111.us-west-2.compute.internal +rate-limiter-1860e37d2-vcth7 0/1 CrashLoopBackOff 14 51m 10.42.1.126 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-8170bec71-832bz 1/1 Running 14 10m 10.42.26.58 ip-10-42-3-96.us-west-2.compute.internal +metrics-ingest-be4215728-atw6n 1/1 Running 0 64h 10.42.20.84 ip-10-42-3-96.us-west-2.compute.internal +notify-dispatch-9f4fe7576-jwfpq 1/1 Running 0 52h 10.42.11.173 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-2c9d23a23-c74py 1/1 Running 0 50h 10.42.25.119 ip-10-42-16-244.us-west-2.compute.internal +ledger-sync-d21335fbb-8q7xp 1/1 Running 1 92h 10.42.28.174 ip-10-42-13-183.us-west-2.compute.internal +auth-gateway-410606fa4-a4qzt 1/1 Running 3 39m 10.42.22.164 ip-10-42-2-206.us-west-2.compute.internal +export-scheduler-6f3761c62-d64rt 1/1 Running 0 54h 10.42.15.45 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-d5e5cb697-rwctc 1/1 Running 3 46m 10.42.31.158 ip-10-42-12-115.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-1-169.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751970801 WARN [ingest-7] checkpoint written offset=2706815 epoch=9 +1751970811 INFO [ingest-1] dropping oversized record bytes=32517 topic=events.raw +1751970817 INFO [ingest-5] gc pause exceeded budget pause_ms=2159 heap_mb=2210 +1751970821 WARN [ingest-4] rebalance triggered generation=1 members=38 +1751970830 INFO [ingest-0] rebalance triggered generation=5 members=21 +1751970839 WARN [ingest-6] gc pause exceeded budget pause_ms=1383 heap_mb=1824 +1751970842 INFO [ingest-1] dropping oversized record bytes=77200 topic=events.raw +1751970850 INFO [ingest-5] slow consumer detected partition=13 lag=3553084 +1751970857 INFO [ingest-3] dropping oversized record bytes=3492 topic=events.raw +1751970867 INFO [ingest-5] dropping oversized record bytes=61014 topic=events.raw +1751970874 INFO [ingest-7] compaction pass complete segments=44 reclaimed_mb=2331 +1751970877 INFO [ingest-5] slow consumer detected partition=19 lag=1709412 +1751970888 INFO [ingest-0] retry attempt=8 for shard=8 after connection reset +1751970894 INFO [ingest-2] checkpoint written offset=3250935 epoch=2 +1751970901 INFO [ingest-5] gc pause exceeded budget pause_ms=1959 heap_mb=2061 +1751970905 ERROR [ingest-1] rebalance triggered generation=2 members=13 +1751970912 WARN [ingest-6] flush batch size=4096 dur_ms=1658 backlog=70444 +1751970924 INFO [ingest-6] flush batch size=4096 dur_ms=468 backlog=59766 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2305 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 3: the flaky `checkout-flow` integration job + +**priya** (13:03): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (13:07): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (13:09): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (13:14): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (13:16): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (13:20): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (13:25): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (13:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (13:32): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (13:38): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (13:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (13:45): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16478969640 6m32s 1h +completed checkout-flow integration main push 16452195475 8m41s 3h +completed checkout-flow integration main push 16446459204 8m12s 5h +completed checkout-flow integration main push 16419504739 7m39s 7h +completed checkout-flow integration main push 16416691581 8m23s 9h +completed checkout-flow integration main push 16458837334 5m56s 11h +completed checkout-flow integration main push 16437362267 11m55s 13h +completed checkout-flow integration main push 16483577704 8m19s 15h +completed checkout-flow integration main push 16415421553 8m37s 17h +failure checkout-flow integration main push 16489604788 9m20s 19h +completed checkout-flow integration main push 16433040918 11m28s 21h +completed checkout-flow integration main push 16409350066 8m42s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 24 total +Tests: 1 failed, 238 passed, 216 total +Time: 207.784 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v3: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v2 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 3 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 3 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 3 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3321: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-425: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-547: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-610: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-704: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 3: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752001604 INFO worker=w07 job=export:883850 chunk=7/12 flushed bytes=205026046 +1752001617 INFO worker=w04 heartbeat ok inflight=26 claimed_total=249 +1752001629 ERROR worker=w14 job=export:881900 upload attempt=7 failed: connection reset by peer, will retry +1752001643 ERROR worker=w10 job=export:882361 upload attempt=12 failed: connection reset by peer, will retry +1752001654 ERROR worker=w12 job=export:884865 upload attempt=1 failed: connection reset by peer, will retry +1752001670 INFO worker=w08 heartbeat ok inflight=29 claimed_total=283 +1752001679 INFO worker=w12 job=export:881685 state=claimed lease_ms=30000 +1752001694 WARN worker=w09 job=export:884793 lease renewal took 1110ms (budget 5000ms) +1752001706 INFO worker=w16 job=export:884655 upload attempt=3 succeeded after retry +1752001722 WARN worker=w01 job=export:884382 lease renewal took 8909ms (budget 5000ms) +1752001731 INFO worker=w04 heartbeat ok inflight=6 claimed_total=30 +1752001744 INFO worker=w06 job=export:882384 state=claimed lease_ms=30000 +1752001758 INFO scheduler tick pending=4749 claimed=13 completed_last_min=15 +1752001774 WARN scheduler queue depth 4829 exceeds soft limit 5000 +1752001782 INFO worker=w09 job=export:883624 rows=373685 bytes=325073746 dur_ms=9429 state=complete +1752001797 WARN scheduler queue depth 7582 exceeds soft limit 5000 +1752001813 INFO worker=w10 job=export:884860 state=claimed lease_ms=30000 +1752001822 WARN scheduler queue depth 4467 exceeds soft limit 5000 +1752001834 INFO worker=w05 job=export:882715 state=claimed lease_ms=30000 +1752001847 INFO scheduler tick pending=7695 claimed=14 completed_last_min=130 +1752001861 ERROR worker=w16 job=export:884019 upload attempt=7 failed: connection reset by peer, will retry +1752001876 INFO worker=w13 job=export:883409 state=claimed lease_ms=30000 +1752001888 WARN scheduler queue depth 1189 exceeds soft limit 5000 +1752001901 WARN worker=w05 job=export:884218 lease renewal took 3586ms (budget 5000ms) +1752001914 DEBUG worker=w02 pool stats idle=14 active=5 waiting=0 +1752001926 ERROR worker=w11 job=export:884268 upload attempt=2 failed: connection reset by peer, will retry +1752001943 INFO worker=w13 job=export:884959 rows=93123 bytes=153336566 dur_ms=4600 state=complete +1752001956 INFO worker=w03 job=export:882942 upload attempt=11 succeeded after retry +1752001967 INFO worker=w06 heartbeat ok inflight=21 claimed_total=104 +1752001981 ERROR worker=w04 job=export:884536 upload attempt=10 failed: connection reset by peer, will retry +1752001993 WARN scheduler queue depth 3788 exceeds soft limit 5000 +1752002004 INFO worker=w05 heartbeat ok inflight=25 claimed_total=116 +1752002021 INFO worker=w01 job=export:881684 rows=358475 bytes=39173081 dur_ms=7354 state=complete +1752002029 INFO worker=w14 job=export:882988 rows=671044 bytes=122667863 dur_ms=3961 state=complete +1752002043 DEBUG worker=w06 pool stats idle=27 active=6 waiting=0 +1752002059 DEBUG worker=w14 pool stats idle=8 active=1 waiting=0 +1752002069 WARN scheduler queue depth 1975 exceeds soft limit 5000 +1752002083 WARN worker=w08 job=export:881104 lease renewal took 479ms (budget 5000ms) +1752002094 WARN scheduler queue depth 7318 exceeds soft limit 5000 +1752002112 INFO worker=w06 heartbeat ok inflight=24 claimed_total=306 +1752002120 INFO worker=w09 job=export:883309 chunk=1/12 flushed bytes=232211615 +1752002136 DEBUG worker=w10 pool stats idle=9 active=1 waiting=0 +1752002146 WARN scheduler queue depth 1582 exceeds soft limit 5000 +1752002161 ERROR worker=w08 job=export:884088 upload attempt=4 failed: connection reset by peer, will retry +1752002173 INFO worker=w13 job=export:882182 rows=167284 bytes=98328034 dur_ms=7194 state=complete +1752002187 WARN scheduler queue depth 8102 exceeds soft limit 5000 +1752002203 ERROR worker=w09 job=export:883755 upload attempt=5 failed: connection reset by peer, will retry +1752002213 INFO worker=w09 job=export:882695 state=claimed lease_ms=30000 +1752002229 INFO worker=w05 job=export:881761 rows=931498 bytes=372404226 dur_ms=9377 state=complete +1752002241 INFO scheduler tick pending=433 claimed=10 completed_last_min=296 +1752002254 INFO worker=w04 heartbeat ok inflight=22 claimed_total=172 +1752002263 INFO scheduler tick pending=1931 claimed=32 completed_last_min=344 +1752002276 WARN scheduler queue depth 7114 exceeds soft limit 5000 +1752002291 INFO worker=w09 job=export:881728 chunk=8/12 flushed bytes=50922476 +1752002302 WARN scheduler queue depth 2269 exceeds soft limit 5000 +1752002318 INFO worker=w06 job=export:881009 chunk=12/12 flushed bytes=220787501 +1752002333 INFO worker=w14 job=export:883218 rows=453188 bytes=236229945 dur_ms=3825 state=complete +1752002341 WARN worker=w16 job=export:884261 lease renewal took 9394ms (budget 5000ms) +1752002357 DEBUG worker=w14 pool stats idle=30 active=10 waiting=0 +1752002369 INFO worker=w14 job=export:882085 upload attempt=4 succeeded after retry +1752002380 DEBUG worker=w07 pool stats idle=32 active=3 waiting=0 +1752002397 INFO scheduler tick pending=6154 claimed=1 completed_last_min=307 +1752002409 ERROR worker=w10 job=export:884262 upload attempt=3 failed: connection reset by peer, will retry +1752002424 ERROR worker=w15 job=export:884309 upload attempt=7 failed: connection reset by peer, will retry +1752002437 INFO worker=w14 job=export:884433 rows=437074 bytes=358059083 dur_ms=7840 state=complete +1752002448 INFO worker=w03 job=export:882772 state=claimed lease_ms=30000 +1752002458 INFO scheduler tick pending=1174 claimed=6 completed_last_min=400 +1752002473 INFO scheduler tick pending=3253 claimed=10 completed_last_min=55 +1752002485 WARN scheduler queue depth 546 exceeds soft limit 5000 +1752002501 ERROR worker=w09 job=export:884397 upload attempt=3 failed: connection reset by peer, will retry +1752002515 DEBUG worker=w14 pool stats idle=16 active=5 waiting=0 +1752002524 WARN scheduler queue depth 5431 exceeds soft limit 5000 +1752002538 WARN scheduler queue depth 4970 exceeds soft limit 5000 +1752002551 INFO worker=w15 job=export:883827 state=claimed lease_ms=30000 +1752002564 WARN scheduler queue depth 4746 exceeds soft limit 5000 +1752002578 DEBUG worker=w08 pool stats idle=19 active=11 waiting=0 +1752002592 ERROR worker=w14 job=export:883606 upload attempt=8 failed: connection reset by peer, will retry +1752002606 INFO worker=w04 job=export:883539 chunk=10/12 flushed bytes=118265331 +1752002617 INFO worker=w04 job=export:881563 upload attempt=9 succeeded after retry +1752002628 INFO worker=w07 job=export:881583 rows=376633 bytes=305918232 dur_ms=8016 state=complete +1752002643 WARN scheduler queue depth 5170 exceeds soft limit 5000 +1752002658 INFO worker=w02 job=export:881153 upload attempt=5 succeeded after retry +1752002667 INFO worker=w07 job=export:884356 chunk=9/12 flushed bytes=151329690 +1752002682 INFO worker=w15 job=export:881018 chunk=11/12 flushed bytes=48778869 +1752002697 INFO worker=w08 job=export:881737 chunk=6/12 flushed bytes=322766827 +1752002709 WARN scheduler queue depth 1331 exceeds soft limit 5000 +1752002720 WARN scheduler queue depth 4056 exceeds soft limit 5000 +1752002733 INFO worker=w14 job=export:881240 state=claimed lease_ms=30000 +1752002749 INFO worker=w06 job=export:882933 rows=4926 bytes=74062129 dur_ms=7886 state=complete +1752002760 INFO worker=w06 job=export:881753 state=claimed lease_ms=30000 +``` + + +## Cluster triage session 4 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +ledger-sync-63032d33d-xgqra 1/1 Running 2 47h 10.42.12.215 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-efaba1d56-dwcsu 1/1 Running 14 32m 10.42.27.144 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-a394884b3-j6yaj 1/1 Running 2 46h 10.42.23.158 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-bd4a95b58-d43wf 1/1 Running 0 77h 10.42.0.143 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-a1ae34602-ppvyt 1/1 Running 14 41m 10.42.20.26 ip-10-42-3-96.us-west-2.compute.internal +notify-dispatch-98ea1e5c2-82amj 1/1 Running 1 76h 10.42.23.68 ip-10-42-15-133.us-west-2.compute.internal +export-scheduler-ca6ea24db-9j794 1/1 Running 2 54h 10.42.9.69 ip-10-42-2-111.us-west-2.compute.internal +export-scheduler-d57fb9a96-ex4gy 1/1 Running 1 38h 10.42.28.161 ip-10-42-2-206.us-west-2.compute.internal +rate-limiter-4f4fd2010-2rruh 1/1 Running 0 68h 10.42.29.147 ip-10-42-15-133.us-west-2.compute.internal +rate-limiter-112cf4d5b-rab7y 1/1 Running 1 54h 10.42.21.164 ip-10-42-15-133.us-west-2.compute.internal +export-scheduler-e3518dd0c-s4g2m 1/1 Running 0 51h 10.42.14.220 ip-10-42-13-183.us-west-2.compute.internal +ledger-sync-ea4d99c30-dnenm 1/1 Running 0 36h 10.42.6.196 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-77e11c994-qqsqy 1/1 Running 1 7h 10.42.3.139 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-dd6557754-6spfu 1/1 Running 1 88h 10.42.9.33 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-fad89eea3-uu8jh 1/1 Running 0 93h 10.42.20.104 ip-10-42-3-96.us-west-2.compute.internal +metrics-ingest-1ea174f5e-hcpgk 0/1 CrashLoopBackOff 14 8m 10.42.12.13 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-a644dfcc1-tp3fb 1/1 Running 0 42h 10.42.13.190 ip-10-42-3-96.us-west-2.compute.internal +search-indexer-767475165-kxsjj 1/1 Running 1 49h 10.42.24.130 ip-10-42-2-111.us-west-2.compute.internal +webhook-relay-91b2d657f-qkhme 1/1 Running 3 30m 10.42.10.48 ip-10-42-27-195.us-west-2.compute.internal +search-indexer-72818cf76-k2m23 1/1 Running 0 15h 10.42.25.153 ip-10-42-2-111.us-west-2.compute.internal +orders-api-a5fb9d4f0-sxxjb 1/1 Running 0 29h 10.42.16.2 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-e02af9e24-usmce 1/1 Running 0 49h 10.42.3.107 ip-10-42-1-169.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-1-169.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751974405 WARN [ingest-7] rebalance triggered generation=9 members=54 +1751974410 INFO [ingest-3] checkpoint written offset=2827491 epoch=1 +1751974415 INFO [ingest-6] dropping oversized record bytes=10592 topic=events.raw +1751974421 INFO [ingest-4] compaction pass complete segments=45 reclaimed_mb=2968 +1751974430 INFO [ingest-4] slow consumer detected partition=31 lag=232142 +1751974437 ERROR [ingest-0] slow consumer detected partition=0 lag=628506 +1751974446 ERROR [ingest-6] slow consumer detected partition=8 lag=2561513 +1751974453 WARN [ingest-3] gc pause exceeded budget pause_ms=760 heap_mb=832 +1751974456 INFO [ingest-5] compaction pass complete segments=8 reclaimed_mb=2212 +1751974467 INFO [ingest-3] compaction pass complete segments=28 reclaimed_mb=2780 +1751974474 INFO [ingest-6] checkpoint written offset=683142 epoch=5 +1751974478 INFO [ingest-7] compaction pass complete segments=27 reclaimed_mb=867 +1751974488 ERROR [ingest-5] flush batch size=4096 dur_ms=1025 backlog=73692 +1751974496 INFO [ingest-6] retry attempt=2 for shard=22 after connection reset +1751974499 ERROR [ingest-0] compaction pass complete segments=44 reclaimed_mb=1858 +1751974507 ERROR [ingest-0] rebalance triggered generation=4 members=16 +1751974514 ERROR [ingest-2] rebalance triggered generation=1 members=5 +1751974519 INFO [ingest-2] compaction pass complete segments=59 reclaimed_mb=1866 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2537 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 4: the flaky `checkout-flow` integration job + +**priya** (14:02): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (14:05): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (14:11): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (14:14): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (14:19): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (14:21): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (14:27): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (14:29): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (14:33): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (14:39): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (14:40): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (14:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16473683824 6m16s 1h +completed checkout-flow integration main push 16410123000 4m27s 3h +completed checkout-flow integration main push 16434413162 9m58s 5h +failure checkout-flow integration main push 16472780472 7m46s 7h +completed checkout-flow integration main push 16496745898 11m56s 9h +completed checkout-flow integration main push 16426598245 7m30s 11h +completed checkout-flow integration main push 16482001427 5m23s 13h +completed checkout-flow integration main push 16420938031 6m13s 15h +failure checkout-flow integration main push 16475589147 11m33s 17h +completed checkout-flow integration main push 16424579105 9m51s 19h +completed checkout-flow integration main push 16434820241 11m11s 21h +completed checkout-flow integration main push 16426846417 5m40s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 28 passed, 27 total +Tests: 1 failed, 189 passed, 221 total +Time: 190.834 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v4: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v3 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 4 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 4 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 4 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3328: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-437: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-590: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-654: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-752: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 4: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752008803 INFO worker=w02 job=export:883155 chunk=7/12 flushed bytes=158383283 +1752008813 INFO worker=w01 job=export:883562 chunk=2/12 flushed bytes=311739619 +1752008830 DEBUG worker=w01 pool stats idle=17 active=4 waiting=0 +1752008840 INFO scheduler tick pending=5689 claimed=11 completed_last_min=74 +1752008854 INFO worker=w13 job=export:882145 upload attempt=3 succeeded after retry +1752008870 INFO worker=w04 job=export:883555 upload attempt=12 succeeded after retry +1752008879 ERROR worker=w14 job=export:882145 upload attempt=9 failed: connection reset by peer, will retry +1752008896 INFO scheduler tick pending=2532 claimed=20 completed_last_min=223 +1752008908 INFO worker=w03 job=export:883910 chunk=7/12 flushed bytes=323743433 +1752008918 WARN worker=w13 job=export:882543 lease renewal took 8139ms (budget 5000ms) +1752008934 ERROR worker=w05 job=export:884944 upload attempt=7 failed: connection reset by peer, will retry +1752008947 INFO worker=w08 job=export:883705 chunk=1/12 flushed bytes=305634469 +1752008956 INFO worker=w04 job=export:884032 chunk=6/12 flushed bytes=318272457 +1752008971 INFO worker=w01 job=export:881157 state=claimed lease_ms=30000 +1752008984 WARN scheduler queue depth 7541 exceeds soft limit 5000 +1752008999 ERROR worker=w02 job=export:881196 upload attempt=7 failed: connection reset by peer, will retry +1752009012 INFO worker=w12 heartbeat ok inflight=9 claimed_total=262 +1752009026 INFO worker=w06 job=export:883331 state=claimed lease_ms=30000 +1752009038 INFO worker=w08 job=export:882894 chunk=8/12 flushed bytes=311604605 +1752009050 WARN scheduler queue depth 1739 exceeds soft limit 5000 +1752009064 WARN scheduler queue depth 5888 exceeds soft limit 5000 +1752009077 DEBUG worker=w13 pool stats idle=19 active=8 waiting=0 +1752009089 ERROR worker=w01 job=export:883559 upload attempt=11 failed: connection reset by peer, will retry +1752009104 INFO worker=w13 job=export:882372 chunk=11/12 flushed bytes=308467603 +1752009115 INFO worker=w01 job=export:884746 state=claimed lease_ms=30000 +1752009129 INFO worker=w12 job=export:883921 rows=340518 bytes=365029798 dur_ms=3713 state=complete +1752009143 INFO worker=w16 heartbeat ok inflight=27 claimed_total=12 +1752009151 WARN scheduler queue depth 2236 exceeds soft limit 5000 +1752009168 INFO worker=w02 job=export:881055 state=claimed lease_ms=30000 +1752009181 WARN scheduler queue depth 4879 exceeds soft limit 5000 +1752009193 DEBUG worker=w07 pool stats idle=18 active=1 waiting=0 +1752009206 WARN scheduler queue depth 3098 exceeds soft limit 5000 +1752009220 ERROR worker=w07 job=export:881026 upload attempt=9 failed: connection reset by peer, will retry +1752009232 INFO worker=w01 heartbeat ok inflight=27 claimed_total=356 +1752009245 DEBUG worker=w13 pool stats idle=21 active=9 waiting=0 +1752009257 WARN worker=w02 job=export:881105 lease renewal took 3913ms (budget 5000ms) +1752009269 INFO worker=w05 heartbeat ok inflight=24 claimed_total=393 +1752009286 INFO scheduler tick pending=5475 claimed=12 completed_last_min=32 +1752009298 ERROR worker=w09 job=export:883560 upload attempt=3 failed: connection reset by peer, will retry +1752009311 ERROR worker=w02 job=export:881486 upload attempt=4 failed: connection reset by peer, will retry +1752009323 WARN worker=w02 job=export:884402 lease renewal took 885ms (budget 5000ms) +1752009334 INFO worker=w05 job=export:882014 upload attempt=10 succeeded after retry +1752009349 INFO scheduler tick pending=5389 claimed=6 completed_last_min=140 +1752009359 INFO worker=w15 job=export:881603 upload attempt=9 succeeded after retry +1752009375 INFO worker=w07 job=export:884231 state=claimed lease_ms=30000 +1752009387 INFO worker=w11 job=export:883999 rows=478498 bytes=270474128 dur_ms=5038 state=complete +1752009401 WARN worker=w16 job=export:884830 lease renewal took 7413ms (budget 5000ms) +1752009412 INFO worker=w11 job=export:881303 chunk=11/12 flushed bytes=189804104 +1752009426 INFO worker=w15 job=export:882339 state=claimed lease_ms=30000 +1752009441 INFO worker=w12 job=export:882995 rows=76036 bytes=309250563 dur_ms=2675 state=complete +1752009451 INFO worker=w09 job=export:883112 upload attempt=11 succeeded after retry +1752009465 INFO worker=w06 job=export:884202 rows=90847 bytes=151490538 dur_ms=177 state=complete +1752009481 INFO worker=w11 heartbeat ok inflight=21 claimed_total=300 +1752009492 WARN scheduler queue depth 4460 exceeds soft limit 5000 +1752009506 INFO worker=w15 job=export:884684 state=claimed lease_ms=30000 +1752009519 WARN scheduler queue depth 2125 exceeds soft limit 5000 +1752009532 WARN scheduler queue depth 7626 exceeds soft limit 5000 +1752009544 INFO worker=w09 job=export:884527 chunk=8/12 flushed bytes=55792685 +1752009554 WARN scheduler queue depth 6656 exceeds soft limit 5000 +1752009569 DEBUG worker=w08 pool stats idle=20 active=11 waiting=0 +1752009584 INFO worker=w10 job=export:883819 rows=447021 bytes=369418817 dur_ms=8671 state=complete +1752009597 DEBUG worker=w16 pool stats idle=11 active=4 waiting=0 +1752009608 INFO worker=w01 job=export:881222 state=claimed lease_ms=30000 +1752009624 WARN worker=w06 job=export:882067 lease renewal took 5454ms (budget 5000ms) +1752009635 ERROR worker=w02 job=export:881082 upload attempt=7 failed: connection reset by peer, will retry +1752009646 INFO worker=w02 job=export:884827 state=claimed lease_ms=30000 +1752009663 ERROR worker=w02 job=export:883347 upload attempt=2 failed: connection reset by peer, will retry +1752009671 INFO worker=w03 job=export:884957 rows=460085 bytes=83296656 dur_ms=307 state=complete +1752009689 INFO worker=w01 job=export:884651 chunk=6/12 flushed bytes=206740117 +1752009697 WARN scheduler queue depth 3381 exceeds soft limit 5000 +1752009712 DEBUG worker=w01 pool stats idle=2 active=1 waiting=0 +1752009727 INFO worker=w01 job=export:884580 state=claimed lease_ms=30000 +1752009741 INFO worker=w02 job=export:884927 chunk=6/12 flushed bytes=360700660 +1752009752 WARN worker=w10 job=export:881426 lease renewal took 2364ms (budget 5000ms) +1752009764 INFO worker=w15 heartbeat ok inflight=25 claimed_total=271 +1752009776 ERROR worker=w15 job=export:881853 upload attempt=10 failed: connection reset by peer, will retry +1752009790 DEBUG worker=w10 pool stats idle=17 active=9 waiting=0 +1752009801 INFO scheduler tick pending=1106 claimed=11 completed_last_min=218 +1752009819 DEBUG worker=w09 pool stats idle=23 active=2 waiting=0 +1752009829 WARN scheduler queue depth 4482 exceeds soft limit 5000 +1752009842 INFO worker=w09 job=export:881846 upload attempt=8 succeeded after retry +1752009853 INFO worker=w09 heartbeat ok inflight=7 claimed_total=55 +1752009867 DEBUG worker=w15 pool stats idle=11 active=7 waiting=0 +1752009882 INFO scheduler tick pending=6276 claimed=13 completed_last_min=96 +1752009896 INFO worker=w12 heartbeat ok inflight=10 claimed_total=262 +1752009910 INFO worker=w04 job=export:883052 upload attempt=3 succeeded after retry +1752009923 INFO worker=w13 heartbeat ok inflight=24 claimed_total=137 +1752009936 DEBUG worker=w06 pool stats idle=1 active=2 waiting=0 +1752009945 INFO worker=w03 job=export:883459 state=claimed lease_ms=30000 +1752009961 INFO worker=w07 job=export:883283 chunk=12/12 flushed bytes=266125512 +``` + + +## Cluster triage session 5 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +auth-gateway-113079d78-ubzw7 1/1 Running 0 78h 10.42.20.202 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-752212bd1-rc5nq 1/1 Running 1 94h 10.42.18.149 ip-10-42-2-111.us-west-2.compute.internal +webhook-relay-c98ad8568-marn7 1/1 Running 2 42h 10.42.21.127 ip-10-42-2-206.us-west-2.compute.internal +auth-gateway-7ef52950e-wa6gc 1/1 Running 0 65h 10.42.10.171 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-2ead619aa-dqmng 1/1 Running 7 24m 10.42.15.227 ip-10-42-1-169.us-west-2.compute.internal +metrics-ingest-9f7cc5edf-vfb9v 1/1 Running 2 34h 10.42.19.14 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-6ea3781b4-zy9t3 1/1 Running 7 51m 10.42.18.15 ip-10-42-2-111.us-west-2.compute.internal +rate-limiter-b8eadc6ea-wmkyy 1/1 Running 2 21h 10.42.17.239 ip-10-42-2-206.us-west-2.compute.internal +orders-api-19e794834-u7mnj 1/1 Running 0 56h 10.42.11.242 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-8cfbedf8f-bh59q 0/1 CrashLoopBackOff 7 54m 10.42.14.124 ip-10-42-15-133.us-west-2.compute.internal +auth-gateway-e0633b5a0-uwsea 1/1 Running 0 52h 10.42.6.82 ip-10-42-13-183.us-west-2.compute.internal +orders-api-fe2ab3e47-xfv9z 1/1 Running 2 46h 10.42.5.249 ip-10-42-27-195.us-west-2.compute.internal +metrics-ingest-bd41654a4-dmeku 1/1 Running 2 62h 10.42.0.124 ip-10-42-2-111.us-west-2.compute.internal +search-indexer-71b52ab44-tn99g 0/1 CrashLoopBackOff 14 11m 10.42.7.205 ip-10-42-27-195.us-west-2.compute.internal +webhook-relay-1599776d1-2bh9m 1/1 Running 0 64h 10.42.13.12 ip-10-42-2-206.us-west-2.compute.internal +orders-api-5dbb23951-ky5eu 1/1 Running 3 11m 10.42.4.84 ip-10-42-1-169.us-west-2.compute.internal +metrics-ingest-fd422723d-mvf28 0/1 CrashLoopBackOff 7 12m 10.42.22.42 ip-10-42-12-115.us-west-2.compute.internal +rate-limiter-831d3630d-2asup 1/1 Running 0 37h 10.42.7.170 ip-10-42-27-195.us-west-2.compute.internal +metrics-ingest-9f2eaa5d0-rjub4 1/1 Running 14 17m 10.42.23.199 ip-10-42-3-96.us-west-2.compute.internal +export-scheduler-ed4fdd1c2-wruyn 1/1 Running 2 10h 10.42.18.190 ip-10-42-2-206.us-west-2.compute.internal +search-indexer-5f17c78fe-8g6rh 1/1 Running 0 39h 10.42.21.29 ip-10-42-27-195.us-west-2.compute.internal +ledger-sync-607d261f1-7z3ue 1/1 Running 1 59h 10.42.23.160 ip-10-42-13-183.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-15-133.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751978002 WARN [ingest-2] rebalance triggered generation=8 members=57 +1751978011 WARN [ingest-4] rebalance triggered generation=5 members=30 +1751978014 INFO [ingest-5] rebalance triggered generation=2 members=22 +1751978025 WARN [ingest-0] rebalance triggered generation=5 members=40 +1751978029 WARN [ingest-6] retry attempt=3 for shard=30 after connection reset +1751978036 ERROR [ingest-6] retry attempt=1 for shard=23 after connection reset +1751978042 INFO [ingest-5] slow consumer detected partition=2 lag=3721144 +1751978049 INFO [ingest-7] dropping oversized record bytes=41692 topic=events.raw +1751978057 INFO [ingest-4] rebalance triggered generation=6 members=19 +1751978068 INFO [ingest-4] rebalance triggered generation=5 members=59 +1751978070 ERROR [ingest-6] dropping oversized record bytes=21427 topic=events.raw +1751978081 ERROR [ingest-1] flush batch size=4096 dur_ms=1570 backlog=43098 +1751978087 INFO [ingest-0] slow consumer detected partition=17 lag=1105948 +1751978096 INFO [ingest-0] dropping oversized record bytes=12647 topic=events.raw +1751978100 WARN [ingest-5] dropping oversized record bytes=50794 topic=events.raw +1751978110 WARN [ingest-5] checkpoint written offset=1930007 epoch=5 +1751978112 WARN [ingest-6] dropping oversized record bytes=66260 topic=events.raw +1751978122 INFO [ingest-3] checkpoint written offset=3424551 epoch=8 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2130 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 5: the flaky `checkout-flow` integration job + +**priya** (15:03): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (15:07): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (15:09): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (15:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (15:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (15:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (15:26): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (15:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (15:34): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (15:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (15:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (15:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16439127077 4m19s 1h +failure checkout-flow integration main push 16499601717 9m58s 3h +completed checkout-flow integration main push 16419720096 9m10s 5h +completed checkout-flow integration main push 16499274740 11m19s 7h +completed checkout-flow integration main push 16452464929 7m13s 9h +failure checkout-flow integration main push 16487231800 11m13s 11h +completed checkout-flow integration main push 16479277155 8m11s 13h +completed checkout-flow integration main push 16455243308 11m52s 15h +completed checkout-flow integration main push 16490091224 6m52s 17h +completed checkout-flow integration main push 16452446301 11m18s 19h +failure checkout-flow integration main push 16445851229 5m37s 21h +completed checkout-flow integration main push 16481929909 4m13s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 23 total +Tests: 1 failed, 238 passed, 220 total +Time: 206.605 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v5: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v4 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 5 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 5 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 5 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3335: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-440: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-585: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-604: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-791: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 5: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752016002 WARN scheduler queue depth 871 exceeds soft limit 5000 +1752016018 INFO worker=w16 job=export:881165 chunk=7/12 flushed bytes=208635882 +1752016028 INFO worker=w02 heartbeat ok inflight=12 claimed_total=286 +1752016042 INFO worker=w09 job=export:884089 upload attempt=5 succeeded after retry +1752016055 INFO scheduler tick pending=5577 claimed=4 completed_last_min=227 +1752016065 INFO worker=w13 job=export:883373 upload attempt=1 succeeded after retry +1752016080 INFO worker=w03 job=export:883279 chunk=10/12 flushed bytes=292972426 +1752016091 INFO worker=w13 job=export:884403 state=claimed lease_ms=30000 +1752016108 DEBUG worker=w06 pool stats idle=29 active=2 waiting=0 +1752016118 INFO worker=w08 job=export:884438 rows=562689 bytes=48174128 dur_ms=4821 state=complete +1752016130 INFO worker=w15 job=export:881640 state=claimed lease_ms=30000 +1752016145 INFO worker=w08 job=export:884953 rows=479311 bytes=311360524 dur_ms=8028 state=complete +1752016160 INFO worker=w03 job=export:884931 upload attempt=4 succeeded after retry +1752016173 WARN worker=w04 job=export:884882 lease renewal took 8550ms (budget 5000ms) +1752016183 INFO worker=w01 heartbeat ok inflight=15 claimed_total=58 +1752016200 DEBUG worker=w03 pool stats idle=7 active=7 waiting=0 +1752016209 WARN worker=w15 job=export:883116 lease renewal took 4397ms (budget 5000ms) +1752016225 INFO worker=w07 heartbeat ok inflight=21 claimed_total=88 +1752016239 INFO scheduler tick pending=8169 claimed=1 completed_last_min=318 +1752016251 WARN worker=w06 job=export:881906 lease renewal took 3617ms (budget 5000ms) +1752016263 WARN worker=w04 job=export:883501 lease renewal took 9144ms (budget 5000ms) +1752016273 DEBUG worker=w02 pool stats idle=16 active=8 waiting=0 +1752016287 INFO scheduler tick pending=7786 claimed=8 completed_last_min=31 +1752016303 INFO worker=w09 job=export:884162 chunk=10/12 flushed bytes=67405563 +1752016317 ERROR worker=w04 job=export:883108 upload attempt=11 failed: connection reset by peer, will retry +1752016327 INFO worker=w05 job=export:881046 chunk=2/12 flushed bytes=185957159 +1752016342 INFO scheduler tick pending=1867 claimed=22 completed_last_min=174 +1752016355 INFO worker=w03 job=export:883823 chunk=8/12 flushed bytes=364092702 +1752016366 INFO worker=w06 heartbeat ok inflight=20 claimed_total=300 +1752016382 INFO worker=w05 job=export:881997 upload attempt=9 succeeded after retry +1752016395 WARN scheduler queue depth 964 exceeds soft limit 5000 +1752016406 INFO worker=w09 job=export:882714 rows=404736 bytes=341417814 dur_ms=3038 state=complete +1752016418 INFO worker=w09 job=export:883838 chunk=12/12 flushed bytes=157766065 +1752016432 DEBUG worker=w12 pool stats idle=5 active=10 waiting=0 +1752016443 WARN scheduler queue depth 5068 exceeds soft limit 5000 +1752016457 INFO worker=w13 job=export:883503 chunk=10/12 flushed bytes=34573554 +1752016470 INFO worker=w07 job=export:882910 upload attempt=8 succeeded after retry +1752016482 ERROR worker=w04 job=export:882984 upload attempt=10 failed: connection reset by peer, will retry +1752016499 INFO worker=w03 job=export:884104 state=claimed lease_ms=30000 +1752016507 INFO scheduler tick pending=2828 claimed=8 completed_last_min=172 +1752016525 INFO worker=w02 heartbeat ok inflight=26 claimed_total=226 +1752016533 INFO worker=w07 job=export:881121 state=claimed lease_ms=30000 +1752016551 WARN worker=w05 job=export:882839 lease renewal took 4254ms (budget 5000ms) +1752016561 WARN worker=w09 job=export:882871 lease renewal took 5609ms (budget 5000ms) +1752016572 ERROR worker=w01 job=export:883264 upload attempt=12 failed: connection reset by peer, will retry +1752016590 ERROR worker=w03 job=export:882019 upload attempt=2 failed: connection reset by peer, will retry +1752016601 INFO worker=w13 job=export:882727 chunk=8/12 flushed bytes=325863750 +1752016615 INFO worker=w09 job=export:884075 upload attempt=3 succeeded after retry +1752016628 INFO worker=w11 job=export:884989 upload attempt=4 succeeded after retry +1752016641 WARN worker=w14 job=export:883986 lease renewal took 5684ms (budget 5000ms) +1752016653 WARN worker=w03 job=export:882924 lease renewal took 3684ms (budget 5000ms) +1752016665 DEBUG worker=w05 pool stats idle=19 active=11 waiting=0 +1752016677 ERROR worker=w10 job=export:882553 upload attempt=6 failed: connection reset by peer, will retry +1752016691 INFO worker=w06 job=export:884238 upload attempt=10 succeeded after retry +1752016705 INFO worker=w02 job=export:882559 chunk=2/12 flushed bytes=370766941 +1752016718 INFO worker=w16 heartbeat ok inflight=12 claimed_total=304 +1752016730 INFO worker=w07 job=export:883596 chunk=5/12 flushed bytes=150665565 +1752016743 INFO worker=w11 job=export:883252 state=claimed lease_ms=30000 +1752016758 ERROR worker=w12 job=export:881503 upload attempt=3 failed: connection reset by peer, will retry +1752016768 ERROR worker=w12 job=export:883402 upload attempt=1 failed: connection reset by peer, will retry +1752016783 ERROR worker=w16 job=export:884247 upload attempt=6 failed: connection reset by peer, will retry +1752016795 INFO worker=w13 job=export:884230 upload attempt=11 succeeded after retry +1752016810 WARN scheduler queue depth 6089 exceeds soft limit 5000 +1752016819 ERROR worker=w16 job=export:882009 upload attempt=4 failed: connection reset by peer, will retry +1752016836 INFO scheduler tick pending=7423 claimed=26 completed_last_min=293 +1752016849 INFO worker=w16 job=export:883909 chunk=12/12 flushed bytes=108976536 +1752016861 INFO worker=w15 job=export:883199 chunk=6/12 flushed bytes=78898401 +1752016874 INFO worker=w02 job=export:882014 rows=151911 bytes=339453211 dur_ms=1130 state=complete +1752016887 INFO worker=w04 job=export:883816 chunk=1/12 flushed bytes=358568624 +1752016897 DEBUG worker=w11 pool stats idle=23 active=9 waiting=0 +1752016910 INFO worker=w07 job=export:884351 rows=869622 bytes=42540263 dur_ms=6075 state=complete +1752016924 INFO worker=w06 job=export:883189 chunk=10/12 flushed bytes=224078782 +1752016937 INFO worker=w12 job=export:884528 chunk=9/12 flushed bytes=195812101 +1752016952 INFO worker=w11 job=export:882478 rows=195795 bytes=40057714 dur_ms=3041 state=complete +1752016965 WARN worker=w03 job=export:882412 lease renewal took 7500ms (budget 5000ms) +1752016978 WARN scheduler queue depth 1811 exceeds soft limit 5000 +1752016993 INFO worker=w16 heartbeat ok inflight=23 claimed_total=83 +1752017001 INFO worker=w02 job=export:881101 rows=114570 bytes=53028324 dur_ms=2085 state=complete +1752017016 WARN scheduler queue depth 3411 exceeds soft limit 5000 +1752017028 DEBUG worker=w01 pool stats idle=11 active=12 waiting=0 +1752017043 ERROR worker=w15 job=export:884609 upload attempt=9 failed: connection reset by peer, will retry +1752017054 INFO worker=w15 job=export:882605 state=claimed lease_ms=30000 +1752017066 WARN worker=w12 job=export:881445 lease renewal took 4048ms (budget 5000ms) +1752017080 WARN worker=w04 job=export:882771 lease renewal took 6977ms (budget 5000ms) +1752017092 INFO worker=w01 job=export:884144 rows=12020 bytes=34999239 dur_ms=9350 state=complete +1752017110 INFO worker=w14 heartbeat ok inflight=32 claimed_total=189 +1752017119 INFO scheduler tick pending=4550 claimed=14 completed_last_min=226 +1752017132 WARN scheduler queue depth 4289 exceeds soft limit 5000 +1752017147 WARN worker=w08 job=export:883767 lease renewal took 9447ms (budget 5000ms) +1752017160 INFO worker=w01 job=export:884104 state=claimed lease_ms=30000 +``` + + +## Cluster triage session 6 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +auth-gateway-2e55fc1b1-nrdb5 1/1 Running 3 35m 10.42.9.198 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-85dc88617-r7g7v 1/1 Running 1 13h 10.42.18.27 ip-10-42-1-169.us-west-2.compute.internal +billing-worker-39214f29c-k7yrn 1/1 Running 1 58h 10.42.0.114 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-754bc3b09-xekzj 0/1 CrashLoopBackOff 7 35m 10.42.9.15 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-831d24401-ure98 0/1 CrashLoopBackOff 14 32m 10.42.29.237 ip-10-42-1-169.us-west-2.compute.internal +metrics-ingest-c3b1b42df-nz9j6 1/1 Running 0 56h 10.42.19.126 ip-10-42-12-115.us-west-2.compute.internal +auth-gateway-3ebc1e118-5pk4s 0/1 CrashLoopBackOff 7 34m 10.42.11.160 ip-10-42-1-169.us-west-2.compute.internal +ledger-sync-3c67f2a48-ndpuj 1/1 Running 3 54m 10.42.21.18 ip-10-42-2-111.us-west-2.compute.internal +search-indexer-6dd994b69-xw2fz 1/1 Running 1 78h 10.42.17.219 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-aa708799e-vnb7f 1/1 Running 0 45h 10.42.23.37 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-70452a74e-j2vkv 1/1 Running 3 50m 10.42.27.218 ip-10-42-13-183.us-west-2.compute.internal +rate-limiter-92f26e823-kq5er 1/1 Running 7 51m 10.42.12.235 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-891a4faa3-kfndh 1/1 Running 14 9m 10.42.21.228 ip-10-42-27-195.us-west-2.compute.internal +orders-api-3fcee1e0b-dgzwb 1/1 Running 1 58h 10.42.22.8 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-e0952884f-dxacb 1/1 Running 0 13h 10.42.24.29 ip-10-42-16-244.us-west-2.compute.internal +export-scheduler-2b3331bdc-wz3gp 1/1 Running 14 48m 10.42.15.52 ip-10-42-13-183.us-west-2.compute.internal +rate-limiter-c3a4a423a-sctzj 1/1 Running 3 50m 10.42.21.5 ip-10-42-13-183.us-west-2.compute.internal +metrics-ingest-575eda315-5hzrx 1/1 Running 0 41h 10.42.18.20 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-672fa9975-8nv97 1/1 Running 0 25h 10.42.10.42 ip-10-42-2-206.us-west-2.compute.internal +search-indexer-edc1d2852-sg9td 1/1 Running 1 11h 10.42.25.202 ip-10-42-2-111.us-west-2.compute.internal +auth-gateway-095e87124-xt4hj 1/1 Running 1 41h 10.42.9.212 ip-10-42-2-206.us-west-2.compute.internal +billing-worker-124d68f71-e3mbv 1/1 Running 0 50h 10.42.16.210 ip-10-42-3-96.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-27-195.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751981602 INFO [ingest-4] retry attempt=8 for shard=11 after connection reset +1751981612 INFO [ingest-3] gc pause exceeded budget pause_ms=675 heap_mb=3501 +1751981616 INFO [ingest-2] checkpoint written offset=2634054 epoch=2 +1751981625 INFO [ingest-5] gc pause exceeded budget pause_ms=2209 heap_mb=3624 +1751981628 INFO [ingest-5] slow consumer detected partition=13 lag=2370475 +1751981640 WARN [ingest-1] compaction pass complete segments=57 reclaimed_mb=1689 +1751981645 INFO [ingest-7] gc pause exceeded budget pause_ms=1712 heap_mb=2138 +1751981651 INFO [ingest-0] flush batch size=4096 dur_ms=1744 backlog=29119 +1751981661 INFO [ingest-6] compaction pass complete segments=64 reclaimed_mb=2178 +1751981668 INFO [ingest-2] retry attempt=6 for shard=18 after connection reset +1751981674 INFO [ingest-0] compaction pass complete segments=54 reclaimed_mb=3192 +1751981680 INFO [ingest-1] gc pause exceeded budget pause_ms=1504 heap_mb=202 +1751981684 INFO [ingest-7] compaction pass complete segments=40 reclaimed_mb=2017 +1751981695 ERROR [ingest-4] flush batch size=4096 dur_ms=212 backlog=53459 +1751981698 INFO [ingest-6] compaction pass complete segments=40 reclaimed_mb=2291 +1751981705 INFO [ingest-0] rebalance triggered generation=4 members=29 +1751981713 INFO [ingest-1] compaction pass complete segments=51 reclaimed_mb=2333 +1751981720 INFO [ingest-0] dropping oversized record bytes=30618 topic=events.raw +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2933 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 6: the flaky `checkout-flow` integration job + +**priya** (16:03): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (16:06): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (16:11): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (16:14): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (16:17): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (16:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (16:24): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (16:28): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (16:34): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (16:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (16:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (16:47): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16486302632 9m34s 1h +completed checkout-flow integration main push 16445269330 4m40s 3h +failure checkout-flow integration main push 16458988638 11m59s 5h +completed checkout-flow integration main push 16424240378 8m59s 7h +completed checkout-flow integration main push 16429152954 10m20s 9h +completed checkout-flow integration main push 16445495347 10m56s 11h +completed checkout-flow integration main push 16478829276 4m52s 13h +completed checkout-flow integration main push 16452781804 10m40s 15h +completed checkout-flow integration main push 16475244728 7m23s 17h +completed checkout-flow integration main push 16409894218 7m21s 19h +completed checkout-flow integration main push 16432044119 7m37s 21h +failure checkout-flow integration main push 16498938668 7m59s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 25 total +Tests: 1 failed, 184 passed, 221 total +Time: 201.830 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v6: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v5 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 6 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 6 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 6 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3342: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-489: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-534: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-671: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-747: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 6: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752023203 INFO worker=w15 job=export:883088 rows=264821 bytes=83637920 dur_ms=6651 state=complete +1752023214 WARN scheduler queue depth 3482 exceeds soft limit 5000 +1752023228 INFO scheduler tick pending=4292 claimed=21 completed_last_min=158 +1752023242 ERROR worker=w04 job=export:884302 upload attempt=1 failed: connection reset by peer, will retry +1752023254 DEBUG worker=w10 pool stats idle=25 active=12 waiting=0 +1752023270 INFO worker=w16 heartbeat ok inflight=29 claimed_total=160 +1752023281 DEBUG worker=w04 pool stats idle=23 active=3 waiting=0 +1752023292 WARN worker=w06 job=export:881295 lease renewal took 8255ms (budget 5000ms) +1752023307 WARN scheduler queue depth 5617 exceeds soft limit 5000 +1752023318 DEBUG worker=w02 pool stats idle=19 active=12 waiting=0 +1752023334 INFO worker=w11 job=export:883025 rows=66957 bytes=31393767 dur_ms=1578 state=complete +1752023347 WARN scheduler queue depth 5809 exceeds soft limit 5000 +1752023357 INFO scheduler tick pending=3341 claimed=2 completed_last_min=149 +1752023370 INFO scheduler tick pending=2356 claimed=28 completed_last_min=44 +1752023384 INFO worker=w12 job=export:883064 state=claimed lease_ms=30000 +1752023395 INFO worker=w01 job=export:881555 rows=451254 bytes=197321108 dur_ms=7510 state=complete +1752023410 INFO worker=w12 job=export:882841 rows=775259 bytes=213619161 dur_ms=3901 state=complete +1752023424 INFO worker=w05 heartbeat ok inflight=17 claimed_total=84 +1752023434 INFO worker=w14 job=export:882676 rows=891121 bytes=248159839 dur_ms=4931 state=complete +1752023448 INFO worker=w02 job=export:882899 chunk=12/12 flushed bytes=35899849 +1752023465 INFO worker=w12 heartbeat ok inflight=12 claimed_total=223 +1752023478 INFO worker=w11 job=export:884922 chunk=5/12 flushed bytes=106145239 +1752023491 INFO worker=w07 job=export:883894 upload attempt=9 succeeded after retry +1752023500 INFO worker=w16 job=export:881107 upload attempt=9 succeeded after retry +1752023513 WARN scheduler queue depth 7027 exceeds soft limit 5000 +1752023526 WARN scheduler queue depth 2245 exceeds soft limit 5000 +1752023540 INFO worker=w03 job=export:883936 state=claimed lease_ms=30000 +1752023552 WARN scheduler queue depth 5429 exceeds soft limit 5000 +1752023568 INFO worker=w15 heartbeat ok inflight=14 claimed_total=146 +1752023577 INFO worker=w10 job=export:882598 upload attempt=7 succeeded after retry +1752023590 INFO worker=w07 job=export:882344 upload attempt=5 succeeded after retry +1752023603 INFO worker=w15 heartbeat ok inflight=27 claimed_total=187 +1752023621 INFO worker=w08 heartbeat ok inflight=28 claimed_total=362 +1752023632 WARN scheduler queue depth 205 exceeds soft limit 5000 +1752023642 INFO worker=w14 job=export:884917 rows=291518 bytes=326028593 dur_ms=2333 state=complete +1752023660 ERROR worker=w07 job=export:884928 upload attempt=9 failed: connection reset by peer, will retry +1752023669 ERROR worker=w06 job=export:884490 upload attempt=6 failed: connection reset by peer, will retry +1752023685 WARN scheduler queue depth 3514 exceeds soft limit 5000 +1752023696 INFO worker=w05 job=export:882232 state=claimed lease_ms=30000 +1752023710 INFO worker=w10 heartbeat ok inflight=6 claimed_total=35 +1752023721 WARN scheduler queue depth 2672 exceeds soft limit 5000 +1752023733 INFO worker=w04 job=export:883978 upload attempt=11 succeeded after retry +1752023746 ERROR worker=w14 job=export:884514 upload attempt=6 failed: connection reset by peer, will retry +1752023760 WARN scheduler queue depth 5013 exceeds soft limit 5000 +1752023776 DEBUG worker=w07 pool stats idle=23 active=9 waiting=0 +1752023790 INFO worker=w14 job=export:881578 rows=55042 bytes=356256087 dur_ms=1237 state=complete +1752023803 INFO worker=w10 job=export:882696 state=claimed lease_ms=30000 +1752023811 INFO worker=w10 job=export:882159 rows=354104 bytes=353810474 dur_ms=1248 state=complete +1752023827 INFO worker=w13 heartbeat ok inflight=30 claimed_total=69 +1752023838 INFO worker=w04 heartbeat ok inflight=5 claimed_total=85 +1752023855 INFO scheduler tick pending=1014 claimed=3 completed_last_min=13 +1752023867 DEBUG worker=w09 pool stats idle=10 active=7 waiting=0 +1752023881 INFO worker=w16 job=export:882977 rows=816275 bytes=64729820 dur_ms=7510 state=complete +1752023893 INFO worker=w04 job=export:882256 rows=577145 bytes=8110912 dur_ms=7200 state=complete +1752023907 ERROR worker=w10 job=export:882505 upload attempt=12 failed: connection reset by peer, will retry +1752023916 DEBUG worker=w04 pool stats idle=31 active=8 waiting=0 +1752023931 INFO scheduler tick pending=4182 claimed=16 completed_last_min=55 +1752023943 WARN worker=w09 job=export:884122 lease renewal took 7277ms (budget 5000ms) +1752023959 ERROR worker=w16 job=export:884623 upload attempt=4 failed: connection reset by peer, will retry +1752023969 ERROR worker=w16 job=export:882905 upload attempt=8 failed: connection reset by peer, will retry +1752023982 INFO worker=w03 job=export:883996 state=claimed lease_ms=30000 +1752023994 DEBUG worker=w05 pool stats idle=11 active=9 waiting=0 +1752024011 INFO worker=w13 job=export:883975 chunk=6/12 flushed bytes=55785457 +1752024022 INFO worker=w13 job=export:882704 chunk=2/12 flushed bytes=262191373 +1752024034 INFO worker=w02 job=export:882385 rows=247928 bytes=22555214 dur_ms=146 state=complete +1752024049 ERROR worker=w16 job=export:884233 upload attempt=5 failed: connection reset by peer, will retry +1752024060 DEBUG worker=w13 pool stats idle=23 active=11 waiting=0 +1752024071 INFO worker=w05 job=export:882939 chunk=11/12 flushed bytes=181023769 +1752024085 INFO worker=w01 job=export:882934 chunk=4/12 flushed bytes=157026782 +1752024100 WARN scheduler queue depth 5990 exceeds soft limit 5000 +1752024113 INFO worker=w04 job=export:883492 state=claimed lease_ms=30000 +1752024128 DEBUG worker=w06 pool stats idle=5 active=7 waiting=0 +1752024137 DEBUG worker=w16 pool stats idle=32 active=6 waiting=0 +1752024151 INFO scheduler tick pending=5963 claimed=3 completed_last_min=18 +1752024164 INFO worker=w10 job=export:882338 state=claimed lease_ms=30000 +1752024179 INFO worker=w08 job=export:881127 state=claimed lease_ms=30000 +1752024193 WARN scheduler queue depth 6815 exceeds soft limit 5000 +1752024205 INFO worker=w01 job=export:884833 upload attempt=10 succeeded after retry +1752024215 ERROR worker=w06 job=export:884754 upload attempt=12 failed: connection reset by peer, will retry +1752024229 INFO worker=w14 job=export:884176 upload attempt=3 succeeded after retry +1752024240 INFO scheduler tick pending=2704 claimed=12 completed_last_min=153 +1752024253 INFO worker=w06 job=export:883413 rows=355724 bytes=160938912 dur_ms=553 state=complete +1752024271 WARN worker=w11 job=export:881524 lease renewal took 6984ms (budget 5000ms) +1752024281 INFO worker=w12 job=export:882946 rows=748512 bytes=31747279 dur_ms=6530 state=complete +1752024297 DEBUG worker=w10 pool stats idle=26 active=8 waiting=0 +1752024306 DEBUG worker=w15 pool stats idle=25 active=6 waiting=0 +1752024323 WARN worker=w09 job=export:881486 lease renewal took 5091ms (budget 5000ms) +1752024333 INFO worker=w07 job=export:881443 state=claimed lease_ms=30000 +1752024345 INFO worker=w02 job=export:881129 chunk=3/12 flushed bytes=261172529 +1752024359 INFO worker=w06 job=export:882147 chunk=9/12 flushed bytes=236697365 +``` + + +## Cluster triage session 7 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +auth-gateway-1ec3f1ca5-xsmk9 1/1 Running 0 14h 10.42.26.166 ip-10-42-15-133.us-west-2.compute.internal +export-scheduler-7e02b0043-cn2cx 1/1 Running 2 20h 10.42.20.183 ip-10-42-12-115.us-west-2.compute.internal +auth-gateway-d32a9bfa5-su89x 1/1 Running 0 24h 10.42.12.204 ip-10-42-16-244.us-west-2.compute.internal +notify-dispatch-bb9793d01-4tdch 1/1 Running 0 40h 10.42.18.164 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-8f5269bcd-cnx5a 1/1 Running 0 25h 10.42.19.135 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-b807c0e5a-j5shu 1/1 Running 2 88h 10.42.24.78 ip-10-42-13-183.us-west-2.compute.internal +notify-dispatch-05a4be0a1-h7q4g 1/1 Running 3 8m 10.42.28.72 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-79633bb11-wndrb 1/1 Running 3 46m 10.42.1.151 ip-10-42-27-195.us-west-2.compute.internal +rate-limiter-0a6f83ba8-ru6nk 1/1 Running 0 96h 10.42.20.202 ip-10-42-13-183.us-west-2.compute.internal +ledger-sync-48a60fa66-fxqa4 1/1 Running 0 9h 10.42.20.70 ip-10-42-27-195.us-west-2.compute.internal +webhook-relay-e886579e1-up67b 1/1 Running 3 8m 10.42.24.237 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-35bf9f379-qysm5 1/1 Running 7 19m 10.42.10.62 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-dacd06da3-w9j6d 1/1 Running 3 46m 10.42.14.28 ip-10-42-12-115.us-west-2.compute.internal +rate-limiter-bb3485c4f-et7cu 1/1 Running 2 70h 10.42.0.230 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-4682ebef3-64zdx 1/1 Running 14 16m 10.42.13.192 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-ca55b2925-kvhdh 1/1 Running 0 50h 10.42.10.97 ip-10-42-1-169.us-west-2.compute.internal +billing-worker-cd47c6d4d-4rctb 1/1 Running 14 29m 10.42.31.213 ip-10-42-13-183.us-west-2.compute.internal +ledger-sync-f31869cd1-j3nvu 1/1 Running 0 94h 10.42.8.126 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-96bba8f22-um3nq 1/1 Running 3 11m 10.42.28.29 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-a99488304-3q7u5 1/1 Running 0 65h 10.42.6.223 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-1fc497b4e-64yqv 1/1 Running 0 52h 10.42.17.41 ip-10-42-2-206.us-west-2.compute.internal +export-scheduler-a32a763fc-2am72 1/1 Running 1 77h 10.42.7.25 ip-10-42-12-115.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-3-96.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751985203 INFO [ingest-1] rebalance triggered generation=8 members=19 +1751985209 WARN [ingest-0] dropping oversized record bytes=76669 topic=events.raw +1751985219 ERROR [ingest-4] dropping oversized record bytes=34525 topic=events.raw +1751985223 INFO [ingest-2] slow consumer detected partition=30 lag=651178 +1751985229 INFO [ingest-7] retry attempt=5 for shard=23 after connection reset +1751985238 INFO [ingest-2] flush batch size=4096 dur_ms=2222 backlog=54603 +1751985243 WARN [ingest-2] flush batch size=4096 dur_ms=821 backlog=59127 +1751985250 INFO [ingest-3] retry attempt=9 for shard=13 after connection reset +1751985259 INFO [ingest-2] flush batch size=4096 dur_ms=280 backlog=87135 +1751985266 INFO [ingest-7] slow consumer detected partition=29 lag=103247 +1751985275 INFO [ingest-3] compaction pass complete segments=51 reclaimed_mb=879 +1751985281 INFO [ingest-3] gc pause exceeded budget pause_ms=167 heap_mb=764 +1751985287 INFO [ingest-1] rebalance triggered generation=2 members=59 +1751985296 INFO [ingest-3] gc pause exceeded budget pause_ms=1887 heap_mb=863 +1751985299 ERROR [ingest-5] flush batch size=4096 dur_ms=1129 backlog=13974 +1751985310 WARN [ingest-6] flush batch size=4096 dur_ms=444 backlog=1436 +1751985314 INFO [ingest-0] flush batch size=4096 dur_ms=416 backlog=23619 +1751985323 ERROR [ingest-2] flush batch size=4096 dur_ms=2398 backlog=42739 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2433 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 7: the flaky `checkout-flow` integration job + +**priya** (17:00): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (17:05): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (17:09): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (17:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (17:19): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (17:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (17:24): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (17:30): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (17:32): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (17:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (17:40): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (17:46): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16491525958 4m26s 1h +completed checkout-flow integration main push 16467868617 11m50s 3h +completed checkout-flow integration main push 16478388906 7m30s 5h +completed checkout-flow integration main push 16497052356 8m42s 7h +completed checkout-flow integration main push 16443624288 6m59s 9h +completed checkout-flow integration main push 16468245446 10m43s 11h +completed checkout-flow integration main push 16490736179 8m12s 13h +completed checkout-flow integration main push 16480306792 6m23s 15h +completed checkout-flow integration main push 16431485774 11m36s 17h +completed checkout-flow integration main push 16481454505 8m36s 19h +completed checkout-flow integration main push 16476147588 4m57s 21h +completed checkout-flow integration main push 16402367701 8m54s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 25 passed, 23 total +Tests: 1 failed, 237 passed, 196 total +Time: 189.346 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v7: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v6 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 7 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 7 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 7 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3349: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-430: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-559: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-620: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-722: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 7: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752030403 WARN scheduler queue depth 672 exceeds soft limit 5000 +1752030413 DEBUG worker=w04 pool stats idle=14 active=9 waiting=0 +1752030426 INFO scheduler tick pending=3699 claimed=13 completed_last_min=22 +1752030442 DEBUG worker=w08 pool stats idle=5 active=5 waiting=0 +1752030452 WARN worker=w13 job=export:884942 lease renewal took 3918ms (budget 5000ms) +1752030470 WARN worker=w13 job=export:882450 lease renewal took 823ms (budget 5000ms) +1752030482 INFO worker=w15 job=export:884526 rows=921793 bytes=120416781 dur_ms=3183 state=complete +1752030492 INFO worker=w15 job=export:882069 upload attempt=8 succeeded after retry +1752030507 INFO worker=w15 job=export:882136 chunk=4/12 flushed bytes=120033661 +1752030521 INFO worker=w06 job=export:883977 chunk=12/12 flushed bytes=310889053 +1752030533 DEBUG worker=w03 pool stats idle=13 active=5 waiting=0 +1752030546 INFO worker=w06 heartbeat ok inflight=15 claimed_total=391 +1752030560 WARN scheduler queue depth 1090 exceeds soft limit 5000 +1752030571 INFO worker=w16 job=export:881399 chunk=12/12 flushed bytes=178836782 +1752030584 WARN scheduler queue depth 6526 exceeds soft limit 5000 +1752030597 INFO worker=w15 job=export:884275 chunk=11/12 flushed bytes=68362167 +1752030611 INFO worker=w14 job=export:883485 state=claimed lease_ms=30000 +1752030625 DEBUG worker=w14 pool stats idle=20 active=11 waiting=0 +1752030637 INFO worker=w09 job=export:883035 upload attempt=11 succeeded after retry +1752030650 INFO worker=w15 heartbeat ok inflight=28 claimed_total=73 +1752030661 INFO worker=w08 job=export:881061 chunk=6/12 flushed bytes=229967158 +1752030678 INFO worker=w10 job=export:881769 chunk=2/12 flushed bytes=74606953 +1752030690 INFO worker=w06 job=export:881634 chunk=9/12 flushed bytes=6991129 +1752030704 INFO worker=w03 heartbeat ok inflight=22 claimed_total=204 +1752030716 ERROR worker=w08 job=export:883186 upload attempt=5 failed: connection reset by peer, will retry +1752030730 INFO worker=w10 job=export:883600 state=claimed lease_ms=30000 +1752030738 INFO worker=w15 job=export:882978 rows=332501 bytes=279483990 dur_ms=8622 state=complete +1752030754 INFO worker=w03 heartbeat ok inflight=15 claimed_total=122 +1752030768 INFO worker=w05 job=export:882450 rows=552786 bytes=220157586 dur_ms=5481 state=complete +1752030777 WARN worker=w03 job=export:882162 lease renewal took 685ms (budget 5000ms) +1752030795 ERROR worker=w07 job=export:883061 upload attempt=11 failed: connection reset by peer, will retry +1752030806 WARN scheduler queue depth 291 exceeds soft limit 5000 +1752030820 WARN worker=w11 job=export:883824 lease renewal took 701ms (budget 5000ms) +1752030831 ERROR worker=w04 job=export:881259 upload attempt=6 failed: connection reset by peer, will retry +1752030844 DEBUG worker=w04 pool stats idle=23 active=11 waiting=0 +1752030857 DEBUG worker=w07 pool stats idle=3 active=1 waiting=0 +1752030871 WARN scheduler queue depth 2233 exceeds soft limit 5000 +1752030881 DEBUG worker=w16 pool stats idle=15 active=1 waiting=0 +1752030896 INFO worker=w01 heartbeat ok inflight=9 claimed_total=134 +1752030912 WARN scheduler queue depth 4671 exceeds soft limit 5000 +1752030920 INFO worker=w10 job=export:882125 rows=78710 bytes=367043719 dur_ms=4752 state=complete +1752030934 INFO worker=w08 heartbeat ok inflight=10 claimed_total=174 +1752030949 INFO worker=w05 job=export:882084 chunk=8/12 flushed bytes=122262483 +1752030959 DEBUG worker=w12 pool stats idle=22 active=1 waiting=0 +1752030975 INFO worker=w07 job=export:884214 rows=579752 bytes=103986908 dur_ms=1550 state=complete +1752030989 WARN scheduler queue depth 4349 exceeds soft limit 5000 +1752031002 INFO worker=w02 job=export:882212 upload attempt=11 succeeded after retry +1752031015 INFO worker=w05 heartbeat ok inflight=5 claimed_total=247 +1752031028 INFO worker=w11 job=export:882745 chunk=8/12 flushed bytes=31957539 +1752031041 WARN worker=w11 job=export:882760 lease renewal took 7207ms (budget 5000ms) +1752031053 WARN worker=w09 job=export:884481 lease renewal took 8207ms (budget 5000ms) +1752031064 WARN scheduler queue depth 3442 exceeds soft limit 5000 +1752031076 WARN scheduler queue depth 3095 exceeds soft limit 5000 +1752031091 INFO worker=w04 job=export:883181 rows=5560 bytes=106180306 dur_ms=1389 state=complete +1752031102 INFO worker=w13 job=export:883553 rows=687786 bytes=180535322 dur_ms=6459 state=complete +1752031120 WARN worker=w08 job=export:884694 lease renewal took 4143ms (budget 5000ms) +1752031130 DEBUG worker=w10 pool stats idle=30 active=10 waiting=0 +1752031142 INFO worker=w07 job=export:881971 state=claimed lease_ms=30000 +1752031154 WARN scheduler queue depth 2075 exceeds soft limit 5000 +1752031170 WARN worker=w10 job=export:881247 lease renewal took 5274ms (budget 5000ms) +1752031184 WARN worker=w02 job=export:882906 lease renewal took 3672ms (budget 5000ms) +1752031194 INFO scheduler tick pending=894 claimed=25 completed_last_min=113 +1752031210 INFO worker=w13 job=export:881375 state=claimed lease_ms=30000 +1752031219 INFO worker=w12 heartbeat ok inflight=31 claimed_total=391 +1752031233 WARN scheduler queue depth 714 exceeds soft limit 5000 +1752031246 INFO worker=w15 job=export:881620 upload attempt=8 succeeded after retry +1752031263 WARN worker=w11 job=export:883864 lease renewal took 7404ms (budget 5000ms) +1752031273 INFO worker=w09 job=export:884063 chunk=7/12 flushed bytes=53799222 +1752031285 WARN worker=w16 job=export:884268 lease renewal took 43ms (budget 5000ms) +1752031299 WARN scheduler queue depth 3470 exceeds soft limit 5000 +1752031314 WARN worker=w05 job=export:882236 lease renewal took 7303ms (budget 5000ms) +1752031326 WARN scheduler queue depth 5083 exceeds soft limit 5000 +1752031338 ERROR worker=w04 job=export:881672 upload attempt=10 failed: connection reset by peer, will retry +1752031353 INFO worker=w13 job=export:884282 upload attempt=9 succeeded after retry +1752031366 WARN worker=w10 job=export:884731 lease renewal took 6406ms (budget 5000ms) +1752031377 INFO worker=w12 heartbeat ok inflight=5 claimed_total=115 +1752031389 WARN scheduler queue depth 2034 exceeds soft limit 5000 +1752031404 INFO worker=w06 heartbeat ok inflight=28 claimed_total=99 +1752031415 INFO worker=w11 job=export:884460 chunk=2/12 flushed bytes=358262233 +1752031431 INFO scheduler tick pending=7882 claimed=31 completed_last_min=271 +1752031440 INFO worker=w11 job=export:884873 upload attempt=4 succeeded after retry +1752031455 ERROR worker=w07 job=export:884263 upload attempt=3 failed: connection reset by peer, will retry +1752031469 DEBUG worker=w13 pool stats idle=6 active=1 waiting=0 +1752031484 INFO worker=w04 heartbeat ok inflight=19 claimed_total=46 +1752031492 WARN worker=w13 job=export:883482 lease renewal took 981ms (budget 5000ms) +1752031507 INFO worker=w11 job=export:882435 chunk=5/12 flushed bytes=319950491 +1752031518 INFO scheduler tick pending=5454 claimed=6 completed_last_min=77 +1752031534 INFO worker=w04 job=export:884571 upload attempt=7 succeeded after retry +1752031544 ERROR worker=w05 job=export:881940 upload attempt=7 failed: connection reset by peer, will retry +1752031562 INFO worker=w09 job=export:882855 state=claimed lease_ms=30000 +``` + + +## Cluster triage session 8 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +webhook-relay-f93d5b9fb-q6ak2 1/1 Running 0 21h 10.42.12.45 ip-10-42-2-206.us-west-2.compute.internal +rate-limiter-e11cb77b5-67mrc 1/1 Running 7 31m 10.42.22.247 ip-10-42-1-169.us-west-2.compute.internal +webhook-relay-3d7108482-9zp8t 1/1 Running 1 39h 10.42.23.245 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-925da54b9-a2wpz 1/1 Running 0 21h 10.42.11.21 ip-10-42-12-115.us-west-2.compute.internal +orders-api-96dff9cca-6ruwf 1/1 Running 0 28h 10.42.4.108 ip-10-42-15-133.us-west-2.compute.internal +billing-worker-f9f79954e-7yffg 1/1 Running 3 37m 10.42.8.250 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-78d38ae27-xezvp 1/1 Running 14 52m 10.42.28.247 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-1965bf508-end8c 1/1 Running 0 54h 10.42.21.121 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-953cfb194-en2gm 1/1 Running 7 9m 10.42.13.50 ip-10-42-12-115.us-west-2.compute.internal +orders-api-5b036c02c-8mjvm 1/1 Running 0 3h 10.42.5.50 ip-10-42-27-195.us-west-2.compute.internal +orders-api-d21e709c7-76rv9 1/1 Running 0 11h 10.42.8.6 ip-10-42-3-96.us-west-2.compute.internal +metrics-ingest-82720c1da-7bsca 1/1 Running 0 64h 10.42.13.55 ip-10-42-2-111.us-west-2.compute.internal +export-scheduler-bee49c4f4-kguhw 1/1 Running 14 42m 10.42.12.84 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-8739011b9-6pa55 0/1 CrashLoopBackOff 7 54m 10.42.24.19 ip-10-42-27-195.us-west-2.compute.internal +export-scheduler-7225acbb1-ph2us 1/1 Running 0 52h 10.42.7.208 ip-10-42-13-183.us-west-2.compute.internal +orders-api-302cb73cc-usw55 1/1 Running 0 15h 10.42.2.61 ip-10-42-15-133.us-west-2.compute.internal +webhook-relay-e16285071-4f9kh 1/1 Running 14 14m 10.42.0.188 ip-10-42-27-195.us-west-2.compute.internal +notify-dispatch-e0687013c-n3r5v 1/1 Running 0 70h 10.42.16.247 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-9b9512c50-fqx92 1/1 Running 0 70h 10.42.4.190 ip-10-42-3-96.us-west-2.compute.internal +export-scheduler-7bdc0c340-kvd69 1/1 Running 7 25m 10.42.18.104 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-8e6ade8d5-uucgr 1/1 Running 1 91h 10.42.6.42 ip-10-42-2-206.us-west-2.compute.internal +auth-gateway-fa14643cf-pdrgf 1/1 Running 2 82h 10.42.27.186 ip-10-42-2-111.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-1-169.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751988805 INFO [ingest-7] dropping oversized record bytes=29699 topic=events.raw +1751988811 INFO [ingest-3] gc pause exceeded budget pause_ms=1400 heap_mb=994 +1751988817 ERROR [ingest-7] flush batch size=4096 dur_ms=1763 backlog=52887 +1751988821 WARN [ingest-0] retry attempt=6 for shard=6 after connection reset +1751988828 INFO [ingest-2] checkpoint written offset=57369 epoch=8 +1751988837 INFO [ingest-5] flush batch size=4096 dur_ms=448 backlog=54911 +1751988846 INFO [ingest-1] rebalance triggered generation=6 members=21 +1751988854 ERROR [ingest-5] gc pause exceeded budget pause_ms=548 heap_mb=1260 +1751988857 INFO [ingest-4] compaction pass complete segments=44 reclaimed_mb=1667 +1751988864 INFO [ingest-6] retry attempt=4 for shard=2 after connection reset +1751988870 WARN [ingest-1] rebalance triggered generation=4 members=17 +1751988881 INFO [ingest-2] flush batch size=4096 dur_ms=749 backlog=21423 +1751988887 WARN [ingest-1] flush batch size=4096 dur_ms=1102 backlog=11374 +1751988892 WARN [ingest-5] checkpoint written offset=206577 epoch=9 +1751988901 ERROR [ingest-0] dropping oversized record bytes=58189 topic=events.raw +1751988906 INFO [ingest-4] compaction pass complete segments=36 reclaimed_mb=651 +1751988914 INFO [ingest-0] retry attempt=6 for shard=10 after connection reset +1751988924 INFO [ingest-4] gc pause exceeded budget pause_ms=1932 heap_mb=1535 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2356 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 8: the flaky `checkout-flow` integration job + +**priya** (18:02): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (18:05): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (18:08): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (18:14): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (18:17): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (18:21): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (18:25): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (18:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (18:35): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (18:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (18:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (18:45): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16425768072 9m32s 1h +failure checkout-flow integration main push 16475311804 5m38s 3h +completed checkout-flow integration main push 16400935109 11m29s 5h +completed checkout-flow integration main push 16402175028 9m31s 7h +failure checkout-flow integration main push 16410178065 8m58s 9h +completed checkout-flow integration main push 16430135703 8m33s 11h +completed checkout-flow integration main push 16410704277 8m24s 13h +failure checkout-flow integration main push 16458903419 4m53s 15h +completed checkout-flow integration main push 16473041955 9m45s 17h +failure checkout-flow integration main push 16490193269 6m42s 19h +completed checkout-flow integration main push 16467844448 9m18s 21h +completed checkout-flow integration main push 16458777783 10m46s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 28 passed, 27 total +Tests: 1 failed, 213 passed, 241 total +Time: 225.836 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v8: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v7 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 8 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 8 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 8 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3356: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-424: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-520: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-672: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-788: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 8: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752037603 INFO worker=w05 heartbeat ok inflight=8 claimed_total=163 +1752037616 INFO worker=w08 job=export:882735 upload attempt=5 succeeded after retry +1752037626 ERROR worker=w05 job=export:884918 upload attempt=10 failed: connection reset by peer, will retry +1752037639 INFO worker=w16 job=export:884966 rows=905637 bytes=215750512 dur_ms=1588 state=complete +1752037656 INFO worker=w10 job=export:884522 rows=643846 bytes=331906853 dur_ms=2354 state=complete +1752037669 INFO worker=w14 job=export:884248 rows=648234 bytes=283535415 dur_ms=4572 state=complete +1752037680 WARN scheduler queue depth 2081 exceeds soft limit 5000 +1752037691 INFO worker=w07 job=export:884947 state=claimed lease_ms=30000 +1752037708 INFO worker=w01 job=export:881586 state=claimed lease_ms=30000 +1752037719 INFO worker=w04 job=export:882310 chunk=1/12 flushed bytes=6757075 +1752037731 WARN worker=w12 job=export:881400 lease renewal took 3825ms (budget 5000ms) +1752037744 DEBUG worker=w15 pool stats idle=25 active=8 waiting=0 +1752037756 WARN worker=w07 job=export:881277 lease renewal took 5474ms (budget 5000ms) +1752037773 WARN worker=w11 job=export:883943 lease renewal took 561ms (budget 5000ms) +1752037787 DEBUG worker=w02 pool stats idle=32 active=4 waiting=0 +1752037799 DEBUG worker=w10 pool stats idle=28 active=1 waiting=0 +1752037813 INFO worker=w15 job=export:883265 chunk=6/12 flushed bytes=40387799 +1752037824 ERROR worker=w01 job=export:882884 upload attempt=7 failed: connection reset by peer, will retry +1752037835 INFO worker=w13 job=export:883592 rows=384771 bytes=112297896 dur_ms=7865 state=complete +1752037848 WARN scheduler queue depth 6630 exceeds soft limit 5000 +1752037863 INFO worker=w10 job=export:884306 upload attempt=1 succeeded after retry +1752037876 INFO scheduler tick pending=7495 claimed=11 completed_last_min=40 +1752037888 WARN worker=w04 job=export:883418 lease renewal took 1480ms (budget 5000ms) +1752037900 WARN scheduler queue depth 713 exceeds soft limit 5000 +1752037914 INFO worker=w13 job=export:884969 upload attempt=9 succeeded after retry +1752037927 DEBUG worker=w13 pool stats idle=1 active=5 waiting=0 +1752037939 INFO worker=w06 job=export:884561 upload attempt=8 succeeded after retry +1752037954 WARN worker=w06 job=export:884780 lease renewal took 7384ms (budget 5000ms) +1752037967 DEBUG worker=w07 pool stats idle=28 active=3 waiting=0 +1752037978 INFO worker=w14 job=export:883879 chunk=5/12 flushed bytes=230601134 +1752037992 INFO scheduler tick pending=3094 claimed=19 completed_last_min=69 +1752038004 WARN worker=w16 job=export:883998 lease renewal took 5048ms (budget 5000ms) +1752038019 INFO worker=w14 job=export:881605 state=claimed lease_ms=30000 +1752038033 WARN scheduler queue depth 918 exceeds soft limit 5000 +1752038044 INFO worker=w07 job=export:884165 state=claimed lease_ms=30000 +1752038059 INFO worker=w09 job=export:884067 chunk=3/12 flushed bytes=337596544 +1752038072 WARN scheduler queue depth 3196 exceeds soft limit 5000 +1752038084 DEBUG worker=w10 pool stats idle=8 active=9 waiting=0 +1752038098 INFO worker=w01 heartbeat ok inflight=27 claimed_total=294 +1752038108 WARN scheduler queue depth 4075 exceeds soft limit 5000 +1752038123 INFO scheduler tick pending=3318 claimed=3 completed_last_min=175 +1752038136 INFO worker=w02 job=export:883361 rows=328706 bytes=29259374 dur_ms=1320 state=complete +1752038146 INFO scheduler tick pending=1110 claimed=9 completed_last_min=236 +1752038159 WARN scheduler queue depth 2146 exceeds soft limit 5000 +1752038173 ERROR worker=w08 job=export:881203 upload attempt=10 failed: connection reset by peer, will retry +1752038189 WARN worker=w10 job=export:883099 lease renewal took 8761ms (budget 5000ms) +1752038201 WARN scheduler queue depth 3109 exceeds soft limit 5000 +1752038214 WARN worker=w02 job=export:884329 lease renewal took 1141ms (budget 5000ms) +1752038225 INFO worker=w03 job=export:883551 rows=192143 bytes=29219523 dur_ms=6900 state=complete +1752038241 INFO worker=w07 job=export:882559 state=claimed lease_ms=30000 +1752038252 INFO scheduler tick pending=5678 claimed=22 completed_last_min=323 +1752038265 INFO worker=w01 job=export:881674 state=claimed lease_ms=30000 +1752038280 WARN scheduler queue depth 1584 exceeds soft limit 5000 +1752038292 INFO worker=w14 job=export:884713 state=claimed lease_ms=30000 +1752038307 INFO worker=w03 job=export:881103 chunk=10/12 flushed bytes=119210102 +1752038319 WARN scheduler queue depth 7468 exceeds soft limit 5000 +1752038333 INFO worker=w04 job=export:883147 upload attempt=3 succeeded after retry +1752038341 WARN worker=w14 job=export:882420 lease renewal took 2385ms (budget 5000ms) +1752038359 INFO worker=w10 job=export:883452 upload attempt=9 succeeded after retry +1752038368 INFO scheduler tick pending=6150 claimed=9 completed_last_min=353 +1752038385 WARN worker=w02 job=export:881778 lease renewal took 6108ms (budget 5000ms) +1752038394 WARN scheduler queue depth 2024 exceeds soft limit 5000 +1752038411 ERROR worker=w01 job=export:884191 upload attempt=7 failed: connection reset by peer, will retry +1752038424 INFO worker=w11 job=export:884726 upload attempt=4 succeeded after retry +1752038436 INFO worker=w05 job=export:882006 rows=353974 bytes=332045666 dur_ms=7871 state=complete +1752038446 DEBUG worker=w05 pool stats idle=3 active=9 waiting=0 +1752038462 INFO scheduler tick pending=536 claimed=26 completed_last_min=148 +1752038471 WARN scheduler queue depth 6068 exceeds soft limit 5000 +1752038485 ERROR worker=w13 job=export:883463 upload attempt=11 failed: connection reset by peer, will retry +1752038498 INFO worker=w07 job=export:883241 state=claimed lease_ms=30000 +1752038514 ERROR worker=w02 job=export:883441 upload attempt=1 failed: connection reset by peer, will retry +1752038527 WARN scheduler queue depth 5069 exceeds soft limit 5000 +1752038537 INFO worker=w14 job=export:884142 state=claimed lease_ms=30000 +1752038554 INFO scheduler tick pending=7502 claimed=23 completed_last_min=166 +1752038564 INFO scheduler tick pending=2763 claimed=22 completed_last_min=77 +1752038576 INFO scheduler tick pending=8024 claimed=13 completed_last_min=49 +1752038593 DEBUG worker=w07 pool stats idle=2 active=4 waiting=0 +1752038604 INFO worker=w03 heartbeat ok inflight=1 claimed_total=74 +1752038616 WARN scheduler queue depth 5657 exceeds soft limit 5000 +1752038630 DEBUG worker=w02 pool stats idle=24 active=7 waiting=0 +1752038643 WARN scheduler queue depth 7289 exceeds soft limit 5000 +1752038655 DEBUG worker=w11 pool stats idle=4 active=6 waiting=0 +1752038668 INFO worker=w09 job=export:882816 state=claimed lease_ms=30000 +1752038682 WARN worker=w13 job=export:883157 lease renewal took 6300ms (budget 5000ms) +1752038694 INFO worker=w12 job=export:881350 rows=330449 bytes=117416817 dur_ms=2646 state=complete +1752038709 DEBUG worker=w01 pool stats idle=32 active=9 waiting=0 +1752038721 WARN worker=w15 job=export:882364 lease renewal took 2238ms (budget 5000ms) +1752038734 INFO worker=w13 job=export:881323 upload attempt=3 succeeded after retry +1752038747 INFO scheduler tick pending=6118 claimed=11 completed_last_min=355 +1752038758 INFO worker=w08 job=export:881323 state=claimed lease_ms=30000 +``` + + +## Cluster triage session 9 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +ledger-sync-5ec666b51-u6tr5 1/1 Running 2 60h 10.42.6.53 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-075b6657d-8exa8 1/1 Running 0 24h 10.42.11.191 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-0d81437e2-6ujr9 1/1 Running 0 4h 10.42.19.13 ip-10-42-15-133.us-west-2.compute.internal +orders-api-1bb59898a-fk893 1/1 Running 0 10h 10.42.30.177 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-b827005a5-2hww9 1/1 Running 0 2h 10.42.13.44 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-7b8aae0f2-jgd8n 0/1 CrashLoopBackOff 7 49m 10.42.8.109 ip-10-42-1-169.us-west-2.compute.internal +notify-dispatch-f7fd7a2e2-en3nt 0/1 CrashLoopBackOff 7 28m 10.42.21.191 ip-10-42-1-169.us-west-2.compute.internal +rate-limiter-3cc703b2d-gr9vh 1/1 Running 2 74h 10.42.29.11 ip-10-42-3-96.us-west-2.compute.internal +orders-api-d23880323-x3twc 1/1 Running 0 94h 10.42.23.236 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-28f874436-7s7zt 1/1 Running 0 36h 10.42.16.132 ip-10-42-12-115.us-west-2.compute.internal +rate-limiter-fd5cb87e8-aqj3m 1/1 Running 3 21m 10.42.18.43 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-7c1ac3cf4-qzxds 1/1 Running 1 93h 10.42.23.103 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-de8b1ad56-qgwbd 1/1 Running 0 33h 10.42.15.210 ip-10-42-12-115.us-west-2.compute.internal +export-scheduler-4577bf5e1-4g5uk 1/1 Running 14 15m 10.42.23.30 ip-10-42-27-195.us-west-2.compute.internal +notify-dispatch-2136d1131-v8n3t 1/1 Running 0 17h 10.42.16.221 ip-10-42-13-183.us-west-2.compute.internal +webhook-relay-1ece5e10e-f23qg 1/1 Running 0 63h 10.42.31.54 ip-10-42-1-169.us-west-2.compute.internal +rate-limiter-332385a80-kn7gh 1/1 Running 0 95h 10.42.23.167 ip-10-42-3-96.us-west-2.compute.internal +billing-worker-3f668db57-mm46e 1/1 Running 7 46m 10.42.13.120 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-85b9f651f-r574a 1/1 Running 3 37m 10.42.8.65 ip-10-42-15-133.us-west-2.compute.internal +metrics-ingest-1f871a005-rpv6d 1/1 Running 2 95h 10.42.19.166 ip-10-42-13-183.us-west-2.compute.internal +search-indexer-fbe4aa2fc-qy8pj 1/1 Running 1 62h 10.42.25.65 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-6cba66ca0-pu2qq 1/1 Running 1 31h 10.42.26.168 ip-10-42-2-206.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-27-195.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751992401 INFO [ingest-7] compaction pass complete segments=23 reclaimed_mb=622 +1751992410 INFO [ingest-3] retry attempt=1 for shard=6 after connection reset +1751992417 INFO [ingest-2] slow consumer detected partition=0 lag=2653417 +1751992422 INFO [ingest-7] dropping oversized record bytes=52983 topic=events.raw +1751992429 INFO [ingest-2] dropping oversized record bytes=32236 topic=events.raw +1751992439 INFO [ingest-7] checkpoint written offset=2488701 epoch=7 +1751992444 INFO [ingest-7] rebalance triggered generation=5 members=29 +1751992453 WARN [ingest-2] compaction pass complete segments=48 reclaimed_mb=1202 +1751992458 INFO [ingest-7] compaction pass complete segments=45 reclaimed_mb=212 +1751992464 INFO [ingest-0] gc pause exceeded budget pause_ms=2047 heap_mb=2668 +1751992473 ERROR [ingest-4] flush batch size=4096 dur_ms=677 backlog=9358 +1751992480 INFO [ingest-2] retry attempt=8 for shard=28 after connection reset +1751992489 INFO [ingest-4] flush batch size=4096 dur_ms=670 backlog=82029 +1751992493 WARN [ingest-2] gc pause exceeded budget pause_ms=558 heap_mb=2115 +1751992498 WARN [ingest-4] flush batch size=4096 dur_ms=2133 backlog=72299 +1751992506 INFO [ingest-7] retry attempt=7 for shard=30 after connection reset +1751992515 INFO [ingest-6] slow consumer detected partition=11 lag=3953186 +1751992523 INFO [ingest-5] compaction pass complete segments=63 reclaimed_mb=1542 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2216 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 9: the flaky `checkout-flow` integration job + +**priya** (19:03): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (19:04): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (19:10): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (19:12): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (19:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (19:23): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (19:27): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (19:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (19:32): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (19:38): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (19:43): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (19:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16485436342 6m35s 1h +completed checkout-flow integration main push 16410338897 6m37s 3h +completed checkout-flow integration main push 16411286549 6m23s 5h +completed checkout-flow integration main push 16464135361 8m27s 7h +completed checkout-flow integration main push 16490519605 6m25s 9h +completed checkout-flow integration main push 16438818679 8m36s 11h +completed checkout-flow integration main push 16449916035 8m11s 13h +completed checkout-flow integration main push 16411030285 8m45s 15h +completed checkout-flow integration main push 16469733029 7m34s 17h +completed checkout-flow integration main push 16465847251 4m27s 19h +completed checkout-flow integration main push 16497816650 6m54s 21h +completed checkout-flow integration main push 16467519292 5m17s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 29 total +Tests: 1 failed, 206 passed, 186 total +Time: 223.109 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v9: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v8 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 9 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 9 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 9 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3363: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-467: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-505: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-674: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-781: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 9: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752044801 INFO worker=w12 job=export:884664 upload attempt=11 succeeded after retry +1752044816 INFO worker=w16 job=export:883602 state=claimed lease_ms=30000 +1752044831 ERROR worker=w09 job=export:881969 upload attempt=1 failed: connection reset by peer, will retry +1752044843 WARN worker=w10 job=export:881966 lease renewal took 578ms (budget 5000ms) +1752044854 INFO worker=w12 job=export:882495 rows=76028 bytes=243862231 dur_ms=9241 state=complete +1752044868 INFO worker=w12 job=export:881377 rows=181625 bytes=255917398 dur_ms=2601 state=complete +1752044878 DEBUG worker=w10 pool stats idle=26 active=12 waiting=0 +1752044894 WARN worker=w07 job=export:883590 lease renewal took 5599ms (budget 5000ms) +1752044908 INFO worker=w14 job=export:882227 rows=216812 bytes=51150687 dur_ms=7305 state=complete +1752044919 INFO worker=w08 job=export:881017 rows=946463 bytes=220571561 dur_ms=8779 state=complete +1752044931 INFO worker=w08 job=export:881763 chunk=10/12 flushed bytes=28894296 +1752044943 INFO worker=w11 job=export:881537 chunk=7/12 flushed bytes=365935969 +1752044956 INFO worker=w07 job=export:882142 rows=805528 bytes=194875620 dur_ms=7259 state=complete +1752044973 INFO worker=w16 job=export:883634 upload attempt=1 succeeded after retry +1752044986 ERROR worker=w05 job=export:884134 upload attempt=7 failed: connection reset by peer, will retry +1752044997 INFO worker=w09 job=export:883066 rows=641517 bytes=314703158 dur_ms=8752 state=complete +1752045012 WARN worker=w12 job=export:884854 lease renewal took 4229ms (budget 5000ms) +1752045022 INFO worker=w12 job=export:883567 rows=755142 bytes=313762889 dur_ms=5807 state=complete +1752045034 ERROR worker=w06 job=export:882540 upload attempt=12 failed: connection reset by peer, will retry +1752045051 WARN scheduler queue depth 1522 exceeds soft limit 5000 +1752045061 INFO worker=w12 job=export:882979 chunk=9/12 flushed bytes=87067342 +1752045075 WARN scheduler queue depth 3456 exceeds soft limit 5000 +1752045086 INFO worker=w14 job=export:884539 state=claimed lease_ms=30000 +1752045100 WARN worker=w10 job=export:881732 lease renewal took 359ms (budget 5000ms) +1752045112 INFO scheduler tick pending=5986 claimed=24 completed_last_min=384 +1752045130 INFO worker=w09 heartbeat ok inflight=29 claimed_total=66 +1752045141 WARN worker=w10 job=export:883934 lease renewal took 450ms (budget 5000ms) +1752045154 INFO worker=w08 heartbeat ok inflight=17 claimed_total=359 +1752045166 INFO worker=w13 heartbeat ok inflight=9 claimed_total=232 +1752045181 INFO worker=w01 job=export:883153 state=claimed lease_ms=30000 +1752045192 WARN worker=w14 job=export:883160 lease renewal took 3908ms (budget 5000ms) +1752045206 INFO worker=w11 job=export:882685 chunk=4/12 flushed bytes=214038861 +1752045219 INFO worker=w02 heartbeat ok inflight=29 claimed_total=293 +1752045230 ERROR worker=w06 job=export:882497 upload attempt=3 failed: connection reset by peer, will retry +1752045246 WARN worker=w15 job=export:883283 lease renewal took 2999ms (budget 5000ms) +1752045257 INFO scheduler tick pending=6526 claimed=30 completed_last_min=209 +1752045271 INFO scheduler tick pending=2630 claimed=31 completed_last_min=11 +1752045281 DEBUG worker=w11 pool stats idle=20 active=2 waiting=0 +1752045297 INFO worker=w02 job=export:882687 upload attempt=2 succeeded after retry +1752045307 WARN worker=w11 job=export:881854 lease renewal took 5788ms (budget 5000ms) +1752045320 INFO worker=w14 heartbeat ok inflight=7 claimed_total=299 +1752045336 INFO worker=w10 job=export:883565 chunk=6/12 flushed bytes=224670103 +1752045349 WARN scheduler queue depth 3604 exceeds soft limit 5000 +1752045364 ERROR worker=w02 job=export:881023 upload attempt=11 failed: connection reset by peer, will retry +1752045373 INFO worker=w08 job=export:883957 state=claimed lease_ms=30000 +1752045388 ERROR worker=w07 job=export:881793 upload attempt=1 failed: connection reset by peer, will retry +1752045401 INFO worker=w08 job=export:882310 state=claimed lease_ms=30000 +1752045416 DEBUG worker=w07 pool stats idle=30 active=5 waiting=0 +1752045429 INFO worker=w03 job=export:881368 state=claimed lease_ms=30000 +1752045439 INFO worker=w15 job=export:881350 upload attempt=5 succeeded after retry +1752045451 WARN worker=w07 job=export:881613 lease renewal took 6819ms (budget 5000ms) +1752045465 INFO worker=w10 job=export:881847 state=claimed lease_ms=30000 +1752045479 INFO worker=w06 heartbeat ok inflight=8 claimed_total=376 +1752045490 INFO worker=w15 job=export:883027 upload attempt=3 succeeded after retry +1752045506 INFO worker=w13 job=export:884780 chunk=10/12 flushed bytes=371973890 +1752045519 INFO worker=w11 job=export:882655 rows=817433 bytes=153035266 dur_ms=8277 state=complete +1752045533 WARN scheduler queue depth 6850 exceeds soft limit 5000 +1752045546 INFO worker=w03 job=export:881051 upload attempt=12 succeeded after retry +1752045556 INFO scheduler tick pending=1272 claimed=27 completed_last_min=189 +1752045569 INFO worker=w08 job=export:882750 rows=91891 bytes=261611569 dur_ms=839 state=complete +1752045581 INFO scheduler tick pending=4190 claimed=19 completed_last_min=29 +1752045597 DEBUG worker=w14 pool stats idle=1 active=4 waiting=0 +1752045607 INFO worker=w03 heartbeat ok inflight=27 claimed_total=12 +1752045624 WARN worker=w16 job=export:881390 lease renewal took 6036ms (budget 5000ms) +1752045634 INFO worker=w14 job=export:884081 chunk=3/12 flushed bytes=275073755 +1752045648 WARN worker=w10 job=export:884892 lease renewal took 1076ms (budget 5000ms) +1752045659 ERROR worker=w02 job=export:881693 upload attempt=6 failed: connection reset by peer, will retry +1752045674 INFO worker=w13 job=export:882817 rows=564551 bytes=99349943 dur_ms=4438 state=complete +1752045684 INFO worker=w09 job=export:884361 chunk=11/12 flushed bytes=34254862 +1752045700 ERROR worker=w01 job=export:882312 upload attempt=4 failed: connection reset by peer, will retry +1752045715 WARN scheduler queue depth 5813 exceeds soft limit 5000 +1752045725 WARN worker=w09 job=export:883048 lease renewal took 3826ms (budget 5000ms) +1752045741 INFO scheduler tick pending=1795 claimed=22 completed_last_min=245 +1752045751 INFO worker=w09 heartbeat ok inflight=31 claimed_total=151 +1752045765 INFO worker=w07 job=export:882713 rows=743835 bytes=28284510 dur_ms=6728 state=complete +1752045776 INFO worker=w16 job=export:883308 upload attempt=12 succeeded after retry +1752045790 INFO scheduler tick pending=7143 claimed=3 completed_last_min=169 +1752045802 INFO worker=w04 heartbeat ok inflight=8 claimed_total=300 +1752045816 ERROR worker=w05 job=export:882681 upload attempt=10 failed: connection reset by peer, will retry +1752045828 WARN worker=w12 job=export:884196 lease renewal took 1710ms (budget 5000ms) +1752045841 WARN scheduler queue depth 6670 exceeds soft limit 5000 +1752045855 INFO worker=w02 job=export:882559 rows=311628 bytes=50531038 dur_ms=4210 state=complete +1752045868 INFO worker=w16 heartbeat ok inflight=16 claimed_total=299 +1752045884 INFO worker=w03 job=export:881836 chunk=3/12 flushed bytes=144008981 +1752045893 DEBUG worker=w05 pool stats idle=32 active=6 waiting=0 +1752045908 INFO worker=w08 job=export:881093 rows=462823 bytes=363008747 dur_ms=7600 state=complete +1752045918 WARN worker=w07 job=export:883976 lease renewal took 9346ms (budget 5000ms) +1752045932 INFO worker=w11 job=export:884204 chunk=9/12 flushed bytes=237524987 +1752045948 INFO worker=w09 job=export:882445 state=claimed lease_ms=30000 +1752045958 WARN scheduler queue depth 1256 exceeds soft limit 5000 +``` + + +## Cluster triage session 10 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +rate-limiter-768707fb3-5s3ra 1/1 Running 1 15h 10.42.1.62 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-8f48c3bce-tbsph 1/1 Running 0 92h 10.42.7.30 ip-10-42-3-96.us-west-2.compute.internal +rate-limiter-6412871c2-wgtg8 1/1 Running 2 83h 10.42.29.177 ip-10-42-16-244.us-west-2.compute.internal +ledger-sync-49cecc6e6-u3n78 0/1 CrashLoopBackOff 7 29m 10.42.24.66 ip-10-42-15-133.us-west-2.compute.internal +orders-api-f684ef4e8-sn9fj 1/1 Running 7 48m 10.42.27.21 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-7fbda11b4-gsfp3 1/1 Running 14 8m 10.42.20.52 ip-10-42-12-115.us-west-2.compute.internal +auth-gateway-e0b4a3828-n8r8x 1/1 Running 1 80h 10.42.9.196 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-a65d48f11-rztmn 1/1 Running 2 77h 10.42.13.222 ip-10-42-1-169.us-west-2.compute.internal +webhook-relay-395aef99d-7svz2 1/1 Running 7 22m 10.42.15.88 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-960d5b6ed-jzfdb 1/1 Running 2 27h 10.42.28.212 ip-10-42-2-206.us-west-2.compute.internal +rate-limiter-7eaa2d406-gk2rp 1/1 Running 0 87h 10.42.23.186 ip-10-42-12-115.us-west-2.compute.internal +webhook-relay-fd5be57dc-z8k7g 0/1 CrashLoopBackOff 7 53m 10.42.14.196 ip-10-42-16-244.us-west-2.compute.internal +rate-limiter-339d30751-heuq7 0/1 CrashLoopBackOff 7 50m 10.42.11.147 ip-10-42-1-169.us-west-2.compute.internal +ledger-sync-39018075e-sdkcm 1/1 Running 0 89h 10.42.20.65 ip-10-42-27-195.us-west-2.compute.internal +export-scheduler-a0a4afe24-vcqa8 1/1 Running 2 92h 10.42.2.248 ip-10-42-15-133.us-west-2.compute.internal +rate-limiter-78edb786c-pabkx 1/1 Running 3 53m 10.42.3.206 ip-10-42-16-244.us-west-2.compute.internal +export-scheduler-8591c47c5-yrp9t 1/1 Running 0 39h 10.42.24.41 ip-10-42-16-244.us-west-2.compute.internal +ledger-sync-97a1c2362-bvcpa 1/1 Running 0 31h 10.42.13.155 ip-10-42-2-111.us-west-2.compute.internal +billing-worker-54ebbcb3b-6xv9u 1/1 Running 3 54m 10.42.6.15 ip-10-42-16-244.us-west-2.compute.internal +export-scheduler-6525db72e-j3f9k 1/1 Running 14 7m 10.42.5.106 ip-10-42-2-206.us-west-2.compute.internal +export-scheduler-959369edb-8wstj 1/1 Running 7 20m 10.42.9.101 ip-10-42-12-115.us-west-2.compute.internal +webhook-relay-61c722994-dye2x 1/1 Running 2 88h 10.42.10.222 ip-10-42-2-111.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-16-244.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751996004 WARN [ingest-5] slow consumer detected partition=5 lag=1690526 +1751996011 WARN [ingest-7] dropping oversized record bytes=76954 topic=events.raw +1751996015 INFO [ingest-0] rebalance triggered generation=8 members=4 +1751996021 INFO [ingest-6] dropping oversized record bytes=22419 topic=events.raw +1751996032 INFO [ingest-2] rebalance triggered generation=9 members=5 +1751996036 INFO [ingest-4] flush batch size=4096 dur_ms=2333 backlog=933 +1751996043 WARN [ingest-7] gc pause exceeded budget pause_ms=2002 heap_mb=503 +1751996054 INFO [ingest-6] gc pause exceeded budget pause_ms=607 heap_mb=1225 +1751996058 INFO [ingest-5] flush batch size=4096 dur_ms=1917 backlog=36919 +1751996063 INFO [ingest-0] checkpoint written offset=3715551 epoch=7 +1751996074 INFO [ingest-0] gc pause exceeded budget pause_ms=1621 heap_mb=2569 +1751996079 ERROR [ingest-4] slow consumer detected partition=20 lag=3176509 +1751996085 WARN [ingest-2] rebalance triggered generation=4 members=48 +1751996091 INFO [ingest-3] dropping oversized record bytes=79930 topic=events.raw +1751996103 INFO [ingest-4] retry attempt=9 for shard=3 after connection reset +1751996110 WARN [ingest-7] rebalance triggered generation=5 members=41 +1751996113 WARN [ingest-4] flush batch size=4096 dur_ms=1477 backlog=58203 +1751996123 INFO [ingest-7] rebalance triggered generation=6 members=64 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2108 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 10: the flaky `checkout-flow` integration job + +**priya** (20:03): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (20:05): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (20:09): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (20:15): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (20:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (20:23): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (20:25): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (20:29): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (20:35): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (20:39): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (20:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (20:47): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16452342669 7m32s 1h +completed checkout-flow integration main push 16485290569 5m19s 3h +completed checkout-flow integration main push 16440908606 8m24s 5h +completed checkout-flow integration main push 16480926417 4m10s 7h +completed checkout-flow integration main push 16472751745 6m47s 9h +completed checkout-flow integration main push 16438671095 6m39s 11h +completed checkout-flow integration main push 16499990154 11m44s 13h +completed checkout-flow integration main push 16401980808 7m16s 15h +completed checkout-flow integration main push 16478388086 8m22s 17h +failure checkout-flow integration main push 16490893421 11m40s 19h +completed checkout-flow integration main push 16459971773 11m50s 21h +failure checkout-flow integration main push 16419860627 9m54s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 28 total +Tests: 1 failed, 212 passed, 195 total +Time: 249.727 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v10: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v9 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 10 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 10 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 10 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3370: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-441: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-599: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-669: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-747: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 10: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752052001 INFO worker=w10 job=export:881575 rows=131062 bytes=224067453 dur_ms=9164 state=complete +1752052018 INFO worker=w16 job=export:884910 state=claimed lease_ms=30000 +1752052029 INFO worker=w15 job=export:881682 upload attempt=8 succeeded after retry +1752052043 INFO scheduler tick pending=5995 claimed=15 completed_last_min=221 +1752052057 INFO worker=w14 job=export:882563 upload attempt=7 succeeded after retry +1752052065 DEBUG worker=w13 pool stats idle=2 active=7 waiting=0 +1752052080 WARN scheduler queue depth 520 exceeds soft limit 5000 +1752052092 INFO worker=w12 heartbeat ok inflight=23 claimed_total=177 +1752052108 INFO worker=w06 job=export:884895 rows=234334 bytes=293744244 dur_ms=2860 state=complete +1752052121 INFO worker=w10 heartbeat ok inflight=1 claimed_total=73 +1752052135 INFO scheduler tick pending=3978 claimed=30 completed_last_min=240 +1752052146 INFO worker=w06 job=export:884201 rows=827849 bytes=304202245 dur_ms=3001 state=complete +1752052160 ERROR worker=w06 job=export:884854 upload attempt=9 failed: connection reset by peer, will retry +1752052174 INFO worker=w14 job=export:884864 chunk=5/12 flushed bytes=238621578 +1752052185 WARN scheduler queue depth 7661 exceeds soft limit 5000 +1752052199 ERROR worker=w10 job=export:881886 upload attempt=12 failed: connection reset by peer, will retry +1752052209 WARN worker=w03 job=export:881494 lease renewal took 1836ms (budget 5000ms) +1752052222 WARN scheduler queue depth 5645 exceeds soft limit 5000 +1752052237 INFO worker=w16 job=export:882798 chunk=1/12 flushed bytes=203641289 +1752052247 INFO worker=w13 heartbeat ok inflight=13 claimed_total=160 +1752052261 INFO worker=w08 job=export:883435 rows=924090 bytes=147290574 dur_ms=536 state=complete +1752052274 INFO worker=w07 job=export:882277 upload attempt=3 succeeded after retry +1752052287 WARN scheduler queue depth 6820 exceeds soft limit 5000 +1752052304 DEBUG worker=w05 pool stats idle=1 active=11 waiting=0 +1752052315 WARN scheduler queue depth 7905 exceeds soft limit 5000 +1752052330 INFO worker=w09 job=export:881607 chunk=3/12 flushed bytes=169992887 +1752052343 INFO scheduler tick pending=3599 claimed=21 completed_last_min=63 +1752052356 INFO worker=w13 job=export:882969 chunk=6/12 flushed bytes=174571536 +1752052369 INFO worker=w10 heartbeat ok inflight=21 claimed_total=165 +1752052381 DEBUG worker=w15 pool stats idle=16 active=5 waiting=0 +1752052391 INFO worker=w01 job=export:884142 state=claimed lease_ms=30000 +1752052407 DEBUG worker=w05 pool stats idle=11 active=8 waiting=0 +1752052419 INFO worker=w06 job=export:882101 chunk=5/12 flushed bytes=64203006 +1752052432 DEBUG worker=w06 pool stats idle=1 active=9 waiting=0 +1752052446 INFO scheduler tick pending=7765 claimed=1 completed_last_min=42 +1752052457 INFO worker=w15 job=export:884658 chunk=8/12 flushed bytes=83468052 +1752052471 INFO scheduler tick pending=1507 claimed=9 completed_last_min=395 +1752052485 INFO scheduler tick pending=5563 claimed=2 completed_last_min=212 +1752052497 WARN worker=w09 job=export:881677 lease renewal took 1596ms (budget 5000ms) +1752052512 INFO worker=w03 job=export:882467 state=claimed lease_ms=30000 +1752052523 WARN worker=w11 job=export:884800 lease renewal took 7639ms (budget 5000ms) +1752052537 WARN scheduler queue depth 5771 exceeds soft limit 5000 +1752052549 WARN worker=w13 job=export:882318 lease renewal took 6985ms (budget 5000ms) +1752052562 INFO worker=w11 job=export:883692 upload attempt=7 succeeded after retry +1752052576 WARN worker=w14 job=export:883468 lease renewal took 8213ms (budget 5000ms) +1752052588 WARN scheduler queue depth 599 exceeds soft limit 5000 +1752052599 DEBUG worker=w03 pool stats idle=17 active=2 waiting=0 +1752052616 WARN worker=w14 job=export:881155 lease renewal took 7253ms (budget 5000ms) +1752052624 WARN scheduler queue depth 2636 exceeds soft limit 5000 +1752052641 WARN worker=w13 job=export:882892 lease renewal took 2906ms (budget 5000ms) +1752052653 INFO worker=w14 job=export:883952 rows=169182 bytes=234266734 dur_ms=2045 state=complete +1752052667 INFO worker=w12 job=export:882184 rows=560231 bytes=220997836 dur_ms=8384 state=complete +1752052676 INFO worker=w04 job=export:882106 upload attempt=3 succeeded after retry +1752052693 DEBUG worker=w06 pool stats idle=18 active=10 waiting=0 +1752052705 DEBUG worker=w05 pool stats idle=26 active=4 waiting=0 +1752052718 WARN worker=w15 job=export:884220 lease renewal took 4885ms (budget 5000ms) +1752052728 INFO worker=w04 job=export:883983 chunk=11/12 flushed bytes=231589060 +1752052741 WARN worker=w08 job=export:883029 lease renewal took 672ms (budget 5000ms) +1752052755 DEBUG worker=w04 pool stats idle=24 active=5 waiting=0 +1752052772 INFO worker=w15 job=export:883103 state=claimed lease_ms=30000 +1752052783 WARN worker=w08 job=export:881513 lease renewal took 7617ms (budget 5000ms) +1752052797 WARN worker=w09 job=export:882983 lease renewal took 884ms (budget 5000ms) +1752052811 INFO worker=w02 job=export:881644 chunk=4/12 flushed bytes=313481002 +1752052819 WARN worker=w01 job=export:881571 lease renewal took 6576ms (budget 5000ms) +1752052835 WARN worker=w14 job=export:882102 lease renewal took 1489ms (budget 5000ms) +1752052847 INFO worker=w04 job=export:882046 upload attempt=11 succeeded after retry +1752052860 INFO scheduler tick pending=4314 claimed=27 completed_last_min=156 +1752052871 WARN scheduler queue depth 3885 exceeds soft limit 5000 +1752052887 WARN scheduler queue depth 263 exceeds soft limit 5000 +1752052898 WARN worker=w01 job=export:882454 lease renewal took 5444ms (budget 5000ms) +1752052915 INFO worker=w08 job=export:882543 rows=390879 bytes=69845534 dur_ms=4666 state=complete +1752052923 DEBUG worker=w08 pool stats idle=4 active=3 waiting=0 +1752052939 INFO worker=w10 job=export:881659 chunk=12/12 flushed bytes=233012719 +1752052952 INFO worker=w03 job=export:883485 upload attempt=1 succeeded after retry +1752052967 INFO worker=w02 heartbeat ok inflight=12 claimed_total=151 +1752052978 WARN worker=w13 job=export:882338 lease renewal took 6359ms (budget 5000ms) +1752052988 INFO worker=w03 job=export:883064 rows=641997 bytes=329760854 dur_ms=3565 state=complete +1752053003 WARN scheduler queue depth 1927 exceeds soft limit 5000 +1752053019 INFO worker=w04 job=export:883745 rows=98840 bytes=367125565 dur_ms=7745 state=complete +1752053029 INFO worker=w08 job=export:883394 upload attempt=5 succeeded after retry +1752053042 INFO worker=w16 job=export:884413 state=claimed lease_ms=30000 +1752053058 WARN scheduler queue depth 220 exceeds soft limit 5000 +1752053071 INFO worker=w03 job=export:883775 rows=438372 bytes=301977503 dur_ms=6894 state=complete +1752053084 ERROR worker=w01 job=export:883075 upload attempt=7 failed: connection reset by peer, will retry +1752053092 INFO worker=w06 job=export:883637 upload attempt=3 succeeded after retry +1752053108 DEBUG worker=w01 pool stats idle=11 active=5 waiting=0 +1752053118 INFO worker=w12 job=export:881802 rows=169601 bytes=190825252 dur_ms=6464 state=complete +1752053136 WARN scheduler queue depth 250 exceeds soft limit 5000 +1752053148 INFO worker=w08 job=export:881942 state=claimed lease_ms=30000 +1752053162 WARN scheduler queue depth 8151 exceeds soft limit 5000 +``` + + +## Cluster triage session 11 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +billing-worker-1e0b735c5-5k4ca 1/1 Running 1 39h 10.42.0.136 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-b079caa8e-fdryz 1/1 Running 2 36h 10.42.18.236 ip-10-42-27-195.us-west-2.compute.internal +metrics-ingest-823597c1a-b2qnp 0/1 CrashLoopBackOff 7 14m 10.42.10.223 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-7a22c2849-2maat 1/1 Running 0 95h 10.42.29.94 ip-10-42-27-195.us-west-2.compute.internal +ledger-sync-c31181b6f-aujxs 1/1 Running 7 9m 10.42.23.123 ip-10-42-13-183.us-west-2.compute.internal +export-scheduler-3ca523fb3-4bdyq 1/1 Running 3 28m 10.42.7.58 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-f41bb362e-j3fvz 1/1 Running 3 25m 10.42.29.132 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-610490e74-33uhr 1/1 Running 14 5m 10.42.25.95 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-61d745979-rkyr9 1/1 Running 0 56h 10.42.10.101 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-2a8fa92d6-jtazg 0/1 CrashLoopBackOff 14 10m 10.42.22.75 ip-10-42-2-111.us-west-2.compute.internal +webhook-relay-8c5ac387c-sb668 0/1 CrashLoopBackOff 14 16m 10.42.18.168 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-7fd55cf5e-6ap8r 1/1 Running 2 18h 10.42.18.106 ip-10-42-16-244.us-west-2.compute.internal +notify-dispatch-26f2eea31-jphc6 1/1 Running 3 6m 10.42.17.65 ip-10-42-27-195.us-west-2.compute.internal +orders-api-63e6de330-kpsvm 1/1 Running 0 5h 10.42.22.58 ip-10-42-3-96.us-west-2.compute.internal +metrics-ingest-99923389a-xt6sd 1/1 Running 0 17h 10.42.2.207 ip-10-42-3-96.us-west-2.compute.internal +ledger-sync-71034f998-b325h 1/1 Running 1 37h 10.42.23.161 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-1692b6d1f-qhdc8 1/1 Running 0 86h 10.42.12.156 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-848047a90-we9q5 1/1 Running 7 53m 10.42.9.190 ip-10-42-12-115.us-west-2.compute.internal +auth-gateway-9e8c66c5b-4j7t7 1/1 Running 0 94h 10.42.3.96 ip-10-42-15-133.us-west-2.compute.internal +export-scheduler-a2c65bc54-zwzk7 1/1 Running 7 39m 10.42.18.209 ip-10-42-13-183.us-west-2.compute.internal +orders-api-b2966e512-hvbkd 1/1 Running 1 58h 10.42.5.89 ip-10-42-13-183.us-west-2.compute.internal +rate-limiter-87a73aa5b-fbr7f 1/1 Running 14 22m 10.42.21.45 ip-10-42-27-195.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-15-133.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1751999601 WARN [ingest-6] flush batch size=4096 dur_ms=2169 backlog=66670 +1751999611 INFO [ingest-2] flush batch size=4096 dur_ms=1241 backlog=61706 +1751999618 INFO [ingest-3] rebalance triggered generation=6 members=45 +1751999624 WARN [ingest-0] dropping oversized record bytes=80881 topic=events.raw +1751999629 INFO [ingest-2] retry attempt=4 for shard=29 after connection reset +1751999640 INFO [ingest-7] checkpoint written offset=2619325 epoch=4 +1751999644 WARN [ingest-3] slow consumer detected partition=13 lag=1039252 +1751999652 INFO [ingest-7] slow consumer detected partition=19 lag=3802201 +1751999658 INFO [ingest-2] checkpoint written offset=2605294 epoch=5 +1751999663 INFO [ingest-2] dropping oversized record bytes=24540 topic=events.raw +1751999673 INFO [ingest-1] retry attempt=8 for shard=1 after connection reset +1751999681 ERROR [ingest-3] slow consumer detected partition=19 lag=1927625 +1751999685 WARN [ingest-1] flush batch size=4096 dur_ms=2210 backlog=87212 +1751999691 INFO [ingest-5] gc pause exceeded budget pause_ms=144 heap_mb=149 +1751999700 WARN [ingest-3] checkpoint written offset=2092324 epoch=9 +1751999709 INFO [ingest-4] flush batch size=4096 dur_ms=2225 backlog=68845 +1751999713 WARN [ingest-1] slow consumer detected partition=26 lag=1514877 +1751999721 INFO [ingest-7] retry attempt=6 for shard=3 after connection reset +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2258 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 11: the flaky `checkout-flow` integration job + +**priya** (21:00): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (21:04): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (21:08): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (21:15): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (21:19): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (21:20): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (21:25): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (21:29): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (21:32): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (21:37): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (21:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (21:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16466232911 5m24s 1h +completed checkout-flow integration main push 16478028470 10m16s 3h +completed checkout-flow integration main push 16427053197 8m36s 5h +completed checkout-flow integration main push 16443215170 6m49s 7h +completed checkout-flow integration main push 16499304527 9m23s 9h +completed checkout-flow integration main push 16424365371 8m31s 11h +completed checkout-flow integration main push 16409932093 9m56s 13h +completed checkout-flow integration main push 16447906804 9m47s 15h +completed checkout-flow integration main push 16468172859 10m29s 17h +completed checkout-flow integration main push 16443596376 4m45s 19h +completed checkout-flow integration main push 16415538279 10m58s 21h +completed checkout-flow integration main push 16449824336 10m53s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 26 total +Tests: 1 failed, 181 passed, 186 total +Time: 208.736 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v11: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v10 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 11 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 11 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 11 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3377: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-477: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-567: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-629: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-722: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 11: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752059203 WARN scheduler queue depth 2387 exceeds soft limit 5000 +1752059215 WARN scheduler queue depth 1946 exceeds soft limit 5000 +1752059229 INFO worker=w16 job=export:882448 chunk=9/12 flushed bytes=166045120 +1752059239 INFO worker=w11 job=export:884051 chunk=12/12 flushed bytes=7724555 +1752059255 INFO worker=w14 job=export:884918 upload attempt=2 succeeded after retry +1752059268 INFO worker=w02 heartbeat ok inflight=18 claimed_total=368 +1752059283 INFO worker=w04 job=export:884081 upload attempt=12 succeeded after retry +1752059294 ERROR worker=w14 job=export:882330 upload attempt=6 failed: connection reset by peer, will retry +1752059304 INFO worker=w11 job=export:883768 upload attempt=9 succeeded after retry +1752059320 INFO worker=w13 job=export:884358 chunk=10/12 flushed bytes=144563881 +1752059334 INFO worker=w03 job=export:884751 rows=698533 bytes=304855440 dur_ms=183 state=complete +1752059343 INFO worker=w02 heartbeat ok inflight=2 claimed_total=272 +1752059356 INFO scheduler tick pending=2818 claimed=24 completed_last_min=36 +1752059374 INFO worker=w11 job=export:882840 chunk=4/12 flushed bytes=81982595 +1752059383 INFO worker=w03 job=export:883100 upload attempt=3 succeeded after retry +1752059398 INFO worker=w11 job=export:881697 rows=460656 bytes=366074424 dur_ms=2526 state=complete +1752059411 ERROR worker=w12 job=export:881853 upload attempt=10 failed: connection reset by peer, will retry +1752059421 INFO worker=w05 job=export:883546 upload attempt=5 succeeded after retry +1752059438 INFO worker=w01 heartbeat ok inflight=6 claimed_total=121 +1752059451 DEBUG worker=w08 pool stats idle=27 active=5 waiting=0 +1752059464 INFO scheduler tick pending=2627 claimed=22 completed_last_min=284 +1752059474 INFO worker=w10 heartbeat ok inflight=8 claimed_total=92 +1752059486 WARN scheduler queue depth 3402 exceeds soft limit 5000 +1752059499 INFO worker=w10 job=export:882326 state=claimed lease_ms=30000 +1752059515 WARN scheduler queue depth 3520 exceeds soft limit 5000 +1752059528 WARN worker=w07 job=export:882363 lease renewal took 289ms (budget 5000ms) +1752059540 WARN scheduler queue depth 7013 exceeds soft limit 5000 +1752059551 INFO worker=w04 heartbeat ok inflight=27 claimed_total=96 +1752059567 WARN scheduler queue depth 417 exceeds soft limit 5000 +1752059577 ERROR worker=w07 job=export:883970 upload attempt=4 failed: connection reset by peer, will retry +1752059594 ERROR worker=w05 job=export:882984 upload attempt=1 failed: connection reset by peer, will retry +1752059604 DEBUG worker=w08 pool stats idle=3 active=1 waiting=0 +1752059617 INFO worker=w02 job=export:882060 upload attempt=3 succeeded after retry +1752059632 ERROR worker=w13 job=export:883038 upload attempt=4 failed: connection reset by peer, will retry +1752059643 INFO worker=w05 heartbeat ok inflight=31 claimed_total=277 +1752059656 WARN worker=w16 job=export:882686 lease renewal took 3718ms (budget 5000ms) +1752059670 WARN scheduler queue depth 6792 exceeds soft limit 5000 +1752059685 INFO worker=w16 job=export:882029 chunk=6/12 flushed bytes=76523117 +1752059695 WARN worker=w09 job=export:884640 lease renewal took 1892ms (budget 5000ms) +1752059707 INFO worker=w05 heartbeat ok inflight=19 claimed_total=15 +1752059722 INFO worker=w01 heartbeat ok inflight=3 claimed_total=17 +1752059738 WARN worker=w15 job=export:883899 lease renewal took 5311ms (budget 5000ms) +1752059746 INFO worker=w03 job=export:881767 chunk=2/12 flushed bytes=351400897 +1752059762 WARN scheduler queue depth 4647 exceeds soft limit 5000 +1752059777 WARN worker=w12 job=export:884477 lease renewal took 7640ms (budget 5000ms) +1752059788 ERROR worker=w12 job=export:882444 upload attempt=10 failed: connection reset by peer, will retry +1752059803 INFO scheduler tick pending=6906 claimed=15 completed_last_min=58 +1752059812 INFO worker=w02 job=export:884689 rows=787958 bytes=87095571 dur_ms=7743 state=complete +1752059827 INFO worker=w02 job=export:881336 upload attempt=8 succeeded after retry +1752059838 WARN scheduler queue depth 4403 exceeds soft limit 5000 +1752059851 INFO scheduler tick pending=6807 claimed=5 completed_last_min=16 +1752059864 ERROR worker=w07 job=export:884915 upload attempt=5 failed: connection reset by peer, will retry +1752059876 WARN worker=w15 job=export:882797 lease renewal took 4279ms (budget 5000ms) +1752059890 INFO worker=w04 job=export:884715 state=claimed lease_ms=30000 +1752059902 WARN worker=w06 job=export:882772 lease renewal took 1594ms (budget 5000ms) +1752059920 INFO worker=w07 job=export:881532 upload attempt=7 succeeded after retry +1752059928 DEBUG worker=w14 pool stats idle=21 active=12 waiting=0 +1752059941 INFO worker=w09 job=export:881627 rows=698497 bytes=255654493 dur_ms=7350 state=complete +1752059956 WARN scheduler queue depth 4819 exceeds soft limit 5000 +1752059972 INFO worker=w09 heartbeat ok inflight=7 claimed_total=240 +1752059981 INFO scheduler tick pending=3896 claimed=12 completed_last_min=390 +1752059994 INFO worker=w06 heartbeat ok inflight=26 claimed_total=86 +1752060006 INFO worker=w10 job=export:884173 upload attempt=5 succeeded after retry +1752060021 DEBUG worker=w02 pool stats idle=19 active=1 waiting=0 +1752060035 DEBUG worker=w11 pool stats idle=28 active=2 waiting=0 +1752060048 WARN worker=w04 job=export:884005 lease renewal took 5211ms (budget 5000ms) +1752060063 WARN scheduler queue depth 1563 exceeds soft limit 5000 +1752060076 INFO worker=w09 job=export:881755 chunk=3/12 flushed bytes=197065838 +1752060089 INFO worker=w08 heartbeat ok inflight=17 claimed_total=312 +1752060098 INFO worker=w11 heartbeat ok inflight=25 claimed_total=42 +1752060110 WARN scheduler queue depth 4964 exceeds soft limit 5000 +1752060124 WARN worker=w13 job=export:883680 lease renewal took 3905ms (budget 5000ms) +1752060141 INFO worker=w09 heartbeat ok inflight=26 claimed_total=129 +1752060152 ERROR worker=w04 job=export:881407 upload attempt=1 failed: connection reset by peer, will retry +1752060163 INFO worker=w05 job=export:884526 state=claimed lease_ms=30000 +1752060180 INFO worker=w09 heartbeat ok inflight=29 claimed_total=27 +1752060193 INFO worker=w09 job=export:883005 state=claimed lease_ms=30000 +1752060203 INFO worker=w16 job=export:882289 upload attempt=3 succeeded after retry +1752060219 INFO worker=w15 job=export:881326 upload attempt=12 succeeded after retry +1752060228 ERROR worker=w07 job=export:881368 upload attempt=3 failed: connection reset by peer, will retry +1752060244 INFO worker=w14 job=export:881682 state=claimed lease_ms=30000 +1752060258 WARN worker=w14 job=export:883865 lease renewal took 2413ms (budget 5000ms) +1752060266 INFO worker=w07 job=export:882140 state=claimed lease_ms=30000 +1752060281 DEBUG worker=w10 pool stats idle=1 active=1 waiting=0 +1752060294 INFO worker=w15 job=export:883632 rows=50432 bytes=135686334 dur_ms=5252 state=complete +1752060305 INFO worker=w10 job=export:882424 chunk=8/12 flushed bytes=77817842 +1752060322 INFO worker=w13 job=export:881234 chunk=5/12 flushed bytes=353918136 +1752060336 WARN scheduler queue depth 6172 exceeds soft limit 5000 +1752060344 INFO scheduler tick pending=6286 claimed=15 completed_last_min=92 +1752060362 DEBUG worker=w10 pool stats idle=28 active=9 waiting=0 +``` + + +## Cluster triage session 12 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +orders-api-c2a122123-5fmak 1/1 Running 0 71h 10.42.2.109 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-5a4e95c7a-a68sg 1/1 Running 2 71h 10.42.5.31 ip-10-42-16-244.us-west-2.compute.internal +ledger-sync-fec020e9f-sgq5y 1/1 Running 3 41m 10.42.1.21 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-a98fc8fe4-q8wmm 1/1 Running 14 11m 10.42.5.57 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-e3d358977-texyy 1/1 Running 3 17m 10.42.9.143 ip-10-42-12-115.us-west-2.compute.internal +ledger-sync-9768c9a43-eu9qg 1/1 Running 0 95h 10.42.21.99 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-22da7423e-5a9ft 0/1 CrashLoopBackOff 14 25m 10.42.23.205 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-39c02d3ad-cft4r 1/1 Running 0 26h 10.42.4.144 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-ff5781e23-zdc76 1/1 Running 1 79h 10.42.20.188 ip-10-42-13-183.us-west-2.compute.internal +notify-dispatch-d5bf102dd-6wwzj 1/1 Running 0 23h 10.42.2.66 ip-10-42-27-195.us-west-2.compute.internal +search-indexer-000655368-2gz8x 1/1 Running 0 90h 10.42.17.35 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-9956cbcf0-vpjgt 1/1 Running 0 85h 10.42.18.172 ip-10-42-2-111.us-west-2.compute.internal +auth-gateway-9789cce1f-446km 1/1 Running 2 63h 10.42.9.54 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-11deb84fb-vmyza 1/1 Running 0 38h 10.42.16.167 ip-10-42-13-183.us-west-2.compute.internal +rate-limiter-3b144a28b-uhanq 1/1 Running 0 32h 10.42.11.171 ip-10-42-2-206.us-west-2.compute.internal +auth-gateway-28cf6ddcd-ger89 1/1 Running 0 22h 10.42.31.181 ip-10-42-1-169.us-west-2.compute.internal +notify-dispatch-aada5f01e-jyztc 0/1 CrashLoopBackOff 14 17m 10.42.26.208 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-800c7fabb-jgz7u 1/1 Running 3 36m 10.42.24.187 ip-10-42-15-133.us-west-2.compute.internal +orders-api-1b2e1d372-ytv8j 0/1 CrashLoopBackOff 7 13m 10.42.7.150 ip-10-42-1-169.us-west-2.compute.internal +orders-api-f25afe0c3-ucmcs 1/1 Running 0 48h 10.42.7.177 ip-10-42-1-169.us-west-2.compute.internal +auth-gateway-895dc2fa8-33xdu 1/1 Running 3 21m 10.42.24.103 ip-10-42-3-96.us-west-2.compute.internal +search-indexer-976f030a4-5d2at 0/1 CrashLoopBackOff 14 37m 10.42.17.208 ip-10-42-13-183.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-15-133.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752003203 INFO [ingest-7] flush batch size=4096 dur_ms=1183 backlog=54861 +1752003207 INFO [ingest-6] slow consumer detected partition=4 lag=2792561 +1752003217 INFO [ingest-4] flush batch size=4096 dur_ms=1626 backlog=81613 +1752003226 INFO [ingest-4] dropping oversized record bytes=53541 topic=events.raw +1752003228 INFO [ingest-5] checkpoint written offset=3577771 epoch=4 +1752003240 INFO [ingest-4] flush batch size=4096 dur_ms=228 backlog=78811 +1752003246 WARN [ingest-2] dropping oversized record bytes=63116 topic=events.raw +1752003254 ERROR [ingest-1] rebalance triggered generation=2 members=5 +1752003257 INFO [ingest-4] checkpoint written offset=3915385 epoch=3 +1752003267 WARN [ingest-2] slow consumer detected partition=28 lag=1830866 +1752003271 INFO [ingest-2] compaction pass complete segments=3 reclaimed_mb=222 +1752003277 INFO [ingest-0] flush batch size=4096 dur_ms=777 backlog=38588 +1752003285 WARN [ingest-4] checkpoint written offset=103283 epoch=1 +1752003291 ERROR [ingest-7] dropping oversized record bytes=1005 topic=events.raw +1752003299 WARN [ingest-6] compaction pass complete segments=40 reclaimed_mb=1934 +1752003308 WARN [ingest-2] gc pause exceeded budget pause_ms=1384 heap_mb=2763 +1752003317 ERROR [ingest-5] slow consumer detected partition=23 lag=2676118 +1752003321 INFO [ingest-5] gc pause exceeded budget pause_ms=1515 heap_mb=1893 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2425 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 12: the flaky `checkout-flow` integration job + +**priya** (22:02): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (22:07): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (22:08): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (22:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (22:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (22:21): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (22:27): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (22:30): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (22:35): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (22:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (22:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (22:46): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16449518456 4m37s 1h +completed checkout-flow integration main push 16453896224 10m49s 3h +completed checkout-flow integration main push 16493014067 9m15s 5h +completed checkout-flow integration main push 16469024699 5m35s 7h +completed checkout-flow integration main push 16404057654 11m33s 9h +failure checkout-flow integration main push 16440739342 4m44s 11h +completed checkout-flow integration main push 16479962813 7m54s 13h +completed checkout-flow integration main push 16471159093 7m46s 15h +completed checkout-flow integration main push 16415842477 6m46s 17h +completed checkout-flow integration main push 16448459187 8m44s 19h +completed checkout-flow integration main push 16474692548 11m44s 21h +completed checkout-flow integration main push 16423103822 8m57s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 28 passed, 28 total +Tests: 1 failed, 213 passed, 208 total +Time: 233.962 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v12: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v11 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 12 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 12 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 12 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3384: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-446: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-540: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-692: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-749: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 12: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752066405 ERROR worker=w08 job=export:883416 upload attempt=7 failed: connection reset by peer, will retry +1752066418 ERROR worker=w01 job=export:884711 upload attempt=8 failed: connection reset by peer, will retry +1752066426 INFO worker=w13 job=export:884613 rows=553714 bytes=268098342 dur_ms=1780 state=complete +1752066441 WARN worker=w04 job=export:884116 lease renewal took 8379ms (budget 5000ms) +1752066452 INFO worker=w10 job=export:881827 rows=33128 bytes=231753576 dur_ms=1470 state=complete +1752066469 DEBUG worker=w12 pool stats idle=11 active=11 waiting=0 +1752066483 INFO worker=w11 heartbeat ok inflight=31 claimed_total=167 +1752066493 DEBUG worker=w04 pool stats idle=11 active=11 waiting=0 +1752066506 WARN worker=w12 job=export:882870 lease renewal took 6863ms (budget 5000ms) +1752066520 ERROR worker=w12 job=export:881718 upload attempt=6 failed: connection reset by peer, will retry +1752066533 INFO scheduler tick pending=4054 claimed=4 completed_last_min=332 +1752066543 WARN scheduler queue depth 1435 exceeds soft limit 5000 +1752066557 DEBUG worker=w14 pool stats idle=12 active=3 waiting=0 +1752066571 DEBUG worker=w11 pool stats idle=12 active=9 waiting=0 +1752066584 DEBUG worker=w11 pool stats idle=3 active=10 waiting=0 +1752066595 DEBUG worker=w08 pool stats idle=14 active=9 waiting=0 +1752066609 INFO worker=w11 job=export:881504 upload attempt=6 succeeded after retry +1752066624 INFO worker=w08 heartbeat ok inflight=9 claimed_total=154 +1752066638 INFO worker=w05 heartbeat ok inflight=4 claimed_total=151 +1752066649 INFO worker=w05 job=export:883024 upload attempt=12 succeeded after retry +1752066664 INFO worker=w14 heartbeat ok inflight=23 claimed_total=209 +1752066673 INFO worker=w14 job=export:884498 state=claimed lease_ms=30000 +1752066691 ERROR worker=w09 job=export:883819 upload attempt=9 failed: connection reset by peer, will retry +1752066699 DEBUG worker=w11 pool stats idle=12 active=12 waiting=0 +1752066717 INFO worker=w06 job=export:884908 upload attempt=4 succeeded after retry +1752066726 WARN scheduler queue depth 1833 exceeds soft limit 5000 +1752066740 DEBUG worker=w13 pool stats idle=31 active=11 waiting=0 +1752066751 INFO worker=w15 job=export:882378 state=claimed lease_ms=30000 +1752066769 WARN scheduler queue depth 7744 exceeds soft limit 5000 +1752066780 INFO worker=w15 job=export:884207 chunk=4/12 flushed bytes=374391991 +1752066790 INFO worker=w08 job=export:882109 state=claimed lease_ms=30000 +1752066806 INFO worker=w11 job=export:884450 chunk=6/12 flushed bytes=191419213 +1752066817 DEBUG worker=w15 pool stats idle=7 active=4 waiting=0 +1752066830 INFO scheduler tick pending=2285 claimed=13 completed_last_min=325 +1752066843 WARN worker=w14 job=export:884200 lease renewal took 4761ms (budget 5000ms) +1752066856 INFO worker=w13 job=export:882161 upload attempt=7 succeeded after retry +1752066873 DEBUG worker=w03 pool stats idle=4 active=6 waiting=0 +1752066882 INFO worker=w04 heartbeat ok inflight=18 claimed_total=84 +1752066896 INFO worker=w08 job=export:882782 rows=257919 bytes=353631449 dur_ms=6786 state=complete +1752066910 INFO scheduler tick pending=4508 claimed=11 completed_last_min=100 +1752066925 DEBUG worker=w04 pool stats idle=6 active=9 waiting=0 +1752066936 INFO worker=w15 job=export:881299 chunk=4/12 flushed bytes=37830416 +1752066947 INFO worker=w10 job=export:883538 chunk=2/12 flushed bytes=72558794 +1752066963 WARN scheduler queue depth 6518 exceeds soft limit 5000 +1752066973 INFO worker=w06 job=export:881686 chunk=9/12 flushed bytes=147005596 +1752066989 INFO worker=w03 heartbeat ok inflight=8 claimed_total=117 +1752066999 INFO worker=w10 job=export:882700 state=claimed lease_ms=30000 +1752067013 INFO worker=w12 job=export:881538 upload attempt=11 succeeded after retry +1752067025 INFO worker=w02 job=export:882674 state=claimed lease_ms=30000 +1752067038 WARN scheduler queue depth 2310 exceeds soft limit 5000 +1752067051 ERROR worker=w05 job=export:884863 upload attempt=6 failed: connection reset by peer, will retry +1752067064 INFO worker=w07 job=export:881468 upload attempt=7 succeeded after retry +1752067078 ERROR worker=w14 job=export:881514 upload attempt=8 failed: connection reset by peer, will retry +1752067092 WARN worker=w14 job=export:884538 lease renewal took 6731ms (budget 5000ms) +1752067106 WARN scheduler queue depth 3974 exceeds soft limit 5000 +1752067119 WARN worker=w03 job=export:883175 lease renewal took 4024ms (budget 5000ms) +1752067128 WARN scheduler queue depth 1836 exceeds soft limit 5000 +1752067146 INFO scheduler tick pending=5728 claimed=13 completed_last_min=156 +1752067154 INFO worker=w01 heartbeat ok inflight=6 claimed_total=116 +1752067172 INFO worker=w05 job=export:881063 chunk=2/12 flushed bytes=346902847 +1752067181 WARN scheduler queue depth 216 exceeds soft limit 5000 +1752067198 INFO worker=w12 job=export:883331 state=claimed lease_ms=30000 +1752067209 DEBUG worker=w12 pool stats idle=9 active=3 waiting=0 +1752067219 INFO worker=w09 heartbeat ok inflight=16 claimed_total=296 +1752067233 INFO worker=w04 job=export:882831 state=claimed lease_ms=30000 +1752067245 INFO worker=w03 job=export:883017 upload attempt=8 succeeded after retry +1752067261 DEBUG worker=w14 pool stats idle=15 active=4 waiting=0 +1752067272 DEBUG worker=w05 pool stats idle=15 active=11 waiting=0 +1752067287 WARN worker=w10 job=export:881076 lease renewal took 952ms (budget 5000ms) +1752067297 INFO worker=w15 job=export:884946 chunk=6/12 flushed bytes=57830954 +1752067315 WARN scheduler queue depth 4193 exceeds soft limit 5000 +1752067324 INFO scheduler tick pending=6287 claimed=13 completed_last_min=207 +1752067338 INFO worker=w03 job=export:884449 state=claimed lease_ms=30000 +1752067351 ERROR worker=w01 job=export:883280 upload attempt=9 failed: connection reset by peer, will retry +1752067367 INFO worker=w09 job=export:881581 state=claimed lease_ms=30000 +1752067379 WARN worker=w06 job=export:881490 lease renewal took 2941ms (budget 5000ms) +1752067389 DEBUG worker=w07 pool stats idle=30 active=12 waiting=0 +1752067402 INFO worker=w09 heartbeat ok inflight=3 claimed_total=101 +1752067419 INFO worker=w02 heartbeat ok inflight=21 claimed_total=341 +1752067431 DEBUG worker=w10 pool stats idle=29 active=8 waiting=0 +1752067444 ERROR worker=w01 job=export:882474 upload attempt=6 failed: connection reset by peer, will retry +1752067454 WARN worker=w08 job=export:883297 lease renewal took 6385ms (budget 5000ms) +1752067466 DEBUG worker=w06 pool stats idle=24 active=10 waiting=0 +1752067484 INFO scheduler tick pending=7242 claimed=4 completed_last_min=157 +1752067496 INFO worker=w03 job=export:883109 state=claimed lease_ms=30000 +1752067506 INFO scheduler tick pending=2919 claimed=20 completed_last_min=99 +1752067518 DEBUG worker=w09 pool stats idle=8 active=1 waiting=0 +1752067535 INFO worker=w01 job=export:883127 upload attempt=9 succeeded after retry +1752067544 DEBUG worker=w14 pool stats idle=28 active=8 waiting=0 +1752067562 ERROR worker=w01 job=export:884251 upload attempt=10 failed: connection reset by peer, will retry +``` + + +## Cluster triage session 13 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +auth-gateway-53c0de6a0-fg3pt 1/1 Running 2 94h 10.42.31.146 ip-10-42-2-206.us-west-2.compute.internal +orders-api-6bd70553d-gmz6q 1/1 Running 1 43h 10.42.18.222 ip-10-42-2-111.us-west-2.compute.internal +export-scheduler-b6d8bc839-nhx79 0/1 CrashLoopBackOff 14 37m 10.42.30.186 ip-10-42-27-195.us-west-2.compute.internal +webhook-relay-735047262-f3czx 1/1 Running 0 14h 10.42.25.16 ip-10-42-27-195.us-west-2.compute.internal +rate-limiter-e48c8b759-bamj8 1/1 Running 0 57h 10.42.12.94 ip-10-42-15-133.us-west-2.compute.internal +webhook-relay-87b3e7e0e-q92vs 1/1 Running 3 19m 10.42.11.117 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-0fada41ba-kxsu8 1/1 Running 3 15m 10.42.14.68 ip-10-42-2-111.us-west-2.compute.internal +webhook-relay-06e1063f6-d4gu9 1/1 Running 1 10h 10.42.16.15 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-8359b75cd-hgz8m 0/1 CrashLoopBackOff 14 47m 10.42.25.230 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-393c9d29f-mmgys 1/1 Running 0 79h 10.42.28.148 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-77b1d19bb-xfa4s 1/1 Running 0 17h 10.42.13.130 ip-10-42-2-206.us-west-2.compute.internal +orders-api-f68bb3590-va6cm 1/1 Running 2 85h 10.42.13.88 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-5428d5e5f-3nfs7 1/1 Running 0 81h 10.42.28.151 ip-10-42-12-115.us-west-2.compute.internal +orders-api-f717ed349-ved9u 1/1 Running 14 38m 10.42.17.113 ip-10-42-1-169.us-west-2.compute.internal +rate-limiter-23c4688e0-d6hj7 0/1 CrashLoopBackOff 14 10m 10.42.31.234 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-4e59b365a-6rnkd 1/1 Running 0 38h 10.42.21.21 ip-10-42-16-244.us-west-2.compute.internal +rate-limiter-31637837b-q5amm 1/1 Running 14 35m 10.42.10.216 ip-10-42-3-96.us-west-2.compute.internal +search-indexer-5c1b3180c-p4wjp 0/1 CrashLoopBackOff 7 30m 10.42.11.84 ip-10-42-12-115.us-west-2.compute.internal +search-indexer-90b3442c4-ajwsu 0/1 CrashLoopBackOff 7 22m 10.42.18.69 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-ef9ea3b92-qph2u 1/1 Running 1 33h 10.42.16.175 ip-10-42-12-115.us-west-2.compute.internal +ledger-sync-77b679a83-n25sf 1/1 Running 2 89h 10.42.29.47 ip-10-42-2-111.us-west-2.compute.internal +orders-api-c834f800e-pxtfp 1/1 Running 14 6m 10.42.29.84 ip-10-42-1-169.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-12-115.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752006805 INFO [ingest-2] gc pause exceeded budget pause_ms=1973 heap_mb=1571 +1752006812 ERROR [ingest-1] slow consumer detected partition=31 lag=2508972 +1752006819 WARN [ingest-2] checkpoint written offset=3577386 epoch=9 +1752006824 WARN [ingest-5] dropping oversized record bytes=25596 topic=events.raw +1752006831 WARN [ingest-4] compaction pass complete segments=55 reclaimed_mb=710 +1752006836 INFO [ingest-7] gc pause exceeded budget pause_ms=1693 heap_mb=1267 +1752006843 INFO [ingest-2] gc pause exceeded budget pause_ms=2220 heap_mb=3601 +1752006853 ERROR [ingest-5] dropping oversized record bytes=11911 topic=events.raw +1752006857 INFO [ingest-1] flush batch size=4096 dur_ms=1647 backlog=87956 +1752006866 WARN [ingest-6] retry attempt=2 for shard=0 after connection reset +1752006875 WARN [ingest-5] slow consumer detected partition=0 lag=1604736 +1752006881 WARN [ingest-7] compaction pass complete segments=28 reclaimed_mb=2825 +1752006888 ERROR [ingest-0] checkpoint written offset=2602490 epoch=7 +1752006895 WARN [ingest-3] slow consumer detected partition=13 lag=2976989 +1752006901 ERROR [ingest-2] gc pause exceeded budget pause_ms=271 heap_mb=1703 +1752006908 INFO [ingest-1] flush batch size=4096 dur_ms=1690 backlog=55798 +1752006912 INFO [ingest-0] slow consumer detected partition=29 lag=412488 +1752006924 INFO [ingest-6] flush batch size=4096 dur_ms=242 backlog=68680 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2590 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 13: the flaky `checkout-flow` integration job + +**priya** (23:00): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (23:06): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (23:09): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (23:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (23:18): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (23:21): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (23:24): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (23:30): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (23:35): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (23:37): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (23:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (23:45): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16473403331 11m33s 1h +completed checkout-flow integration main push 16449576294 6m42s 3h +completed checkout-flow integration main push 16412629134 10m34s 5h +completed checkout-flow integration main push 16473768324 9m48s 7h +completed checkout-flow integration main push 16405770695 11m40s 9h +failure checkout-flow integration main push 16400639104 8m44s 11h +completed checkout-flow integration main push 16412083926 10m35s 13h +failure checkout-flow integration main push 16480003826 9m41s 15h +failure checkout-flow integration main push 16496669041 11m34s 17h +completed checkout-flow integration main push 16483878072 11m51s 19h +completed checkout-flow integration main push 16451569277 10m16s 21h +failure checkout-flow integration main push 16488485831 9m34s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 23 passed, 28 total +Tests: 1 failed, 187 passed, 203 total +Time: 180.368 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v13: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v12 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 13 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 13 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 13 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3391: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-498: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-501: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-658: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-796: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 13: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752073604 INFO worker=w08 job=export:883049 rows=495982 bytes=39100433 dur_ms=7376 state=complete +1752073615 WARN worker=w01 job=export:883667 lease renewal took 374ms (budget 5000ms) +1752073627 INFO worker=w03 job=export:882210 chunk=1/12 flushed bytes=344610798 +1752073642 DEBUG worker=w11 pool stats idle=24 active=1 waiting=0 +1752073655 INFO worker=w16 job=export:881012 state=claimed lease_ms=30000 +1752073669 INFO worker=w08 heartbeat ok inflight=28 claimed_total=26 +1752073683 INFO worker=w02 job=export:883617 state=claimed lease_ms=30000 +1752073694 INFO worker=w15 job=export:882621 chunk=1/12 flushed bytes=351152810 +1752073704 INFO worker=w14 job=export:884704 upload attempt=7 succeeded after retry +1752073719 INFO worker=w03 job=export:884396 rows=409400 bytes=277971754 dur_ms=9477 state=complete +1752073732 INFO scheduler tick pending=1320 claimed=19 completed_last_min=105 +1752073745 INFO worker=w02 job=export:884191 upload attempt=11 succeeded after retry +1752073761 INFO worker=w15 job=export:883521 upload attempt=4 succeeded after retry +1752073773 INFO scheduler tick pending=4700 claimed=23 completed_last_min=231 +1752073785 WARN worker=w14 job=export:884481 lease renewal took 3979ms (budget 5000ms) +1752073797 INFO worker=w08 heartbeat ok inflight=23 claimed_total=381 +1752073808 DEBUG worker=w11 pool stats idle=7 active=5 waiting=0 +1752073825 ERROR worker=w03 job=export:882919 upload attempt=12 failed: connection reset by peer, will retry +1752073836 WARN scheduler queue depth 7248 exceeds soft limit 5000 +1752073848 INFO worker=w06 job=export:882259 upload attempt=6 succeeded after retry +1752073861 DEBUG worker=w04 pool stats idle=1 active=6 waiting=0 +1752073875 WARN scheduler queue depth 1625 exceeds soft limit 5000 +1752073888 DEBUG worker=w14 pool stats idle=18 active=12 waiting=0 +1752073904 ERROR worker=w11 job=export:884067 upload attempt=11 failed: connection reset by peer, will retry +1752073912 INFO worker=w05 heartbeat ok inflight=23 claimed_total=183 +1752073928 INFO worker=w13 job=export:882790 rows=311957 bytes=110717504 dur_ms=4176 state=complete +1752073939 INFO worker=w11 job=export:881889 state=claimed lease_ms=30000 +1752073951 INFO worker=w16 job=export:883844 chunk=6/12 flushed bytes=178969337 +1752073966 INFO worker=w11 job=export:884468 state=claimed lease_ms=30000 +1752073978 INFO worker=w08 job=export:883031 rows=405688 bytes=135372050 dur_ms=4595 state=complete +1752073992 ERROR worker=w13 job=export:884520 upload attempt=12 failed: connection reset by peer, will retry +1752074004 INFO worker=w16 job=export:882248 chunk=5/12 flushed bytes=267726080 +1752074018 INFO worker=w16 heartbeat ok inflight=25 claimed_total=296 +1752074032 INFO worker=w14 job=export:881116 upload attempt=2 succeeded after retry +1752074042 INFO worker=w06 job=export:884352 rows=746864 bytes=323574999 dur_ms=9367 state=complete +1752074058 INFO scheduler tick pending=6256 claimed=21 completed_last_min=279 +1752074072 WARN scheduler queue depth 5409 exceeds soft limit 5000 +1752074084 WARN worker=w01 job=export:883301 lease renewal took 2700ms (budget 5000ms) +1752074099 INFO worker=w10 job=export:881727 rows=76880 bytes=350403226 dur_ms=7417 state=complete +1752074111 INFO worker=w05 job=export:884366 state=claimed lease_ms=30000 +1752074121 INFO scheduler tick pending=8065 claimed=17 completed_last_min=153 +1752074136 INFO worker=w02 heartbeat ok inflight=30 claimed_total=141 +1752074150 INFO worker=w14 job=export:882243 upload attempt=8 succeeded after retry +1752074162 DEBUG worker=w14 pool stats idle=11 active=3 waiting=0 +1752074173 WARN scheduler queue depth 5832 exceeds soft limit 5000 +1752074187 INFO scheduler tick pending=4331 claimed=13 completed_last_min=354 +1752074198 INFO worker=w05 job=export:882341 chunk=8/12 flushed bytes=58796346 +1752074212 DEBUG worker=w03 pool stats idle=11 active=7 waiting=0 +1752074227 INFO worker=w02 job=export:883021 upload attempt=5 succeeded after retry +1752074238 ERROR worker=w01 job=export:883157 upload attempt=9 failed: connection reset by peer, will retry +1752074252 INFO scheduler tick pending=4806 claimed=4 completed_last_min=268 +1752074264 INFO scheduler tick pending=6372 claimed=29 completed_last_min=333 +1752074279 INFO worker=w05 job=export:883732 upload attempt=7 succeeded after retry +1752074291 INFO worker=w09 job=export:881924 state=claimed lease_ms=30000 +1752074303 INFO worker=w14 heartbeat ok inflight=27 claimed_total=217 +1752074319 INFO worker=w06 job=export:882387 state=claimed lease_ms=30000 +1752074328 INFO worker=w16 heartbeat ok inflight=31 claimed_total=354 +1752074342 WARN worker=w08 job=export:882442 lease renewal took 5472ms (budget 5000ms) +1752074354 INFO worker=w12 job=export:883371 state=claimed lease_ms=30000 +1752074371 INFO scheduler tick pending=3442 claimed=23 completed_last_min=96 +1752074385 WARN worker=w12 job=export:882294 lease renewal took 4471ms (budget 5000ms) +1752074394 INFO worker=w07 job=export:881062 state=claimed lease_ms=30000 +1752074407 INFO worker=w02 job=export:883519 rows=12637 bytes=316052257 dur_ms=7951 state=complete +1752074421 INFO worker=w03 heartbeat ok inflight=7 claimed_total=14 +1752074434 ERROR worker=w02 job=export:883001 upload attempt=8 failed: connection reset by peer, will retry +1752074449 INFO scheduler tick pending=2681 claimed=24 completed_last_min=19 +1752074459 INFO worker=w04 job=export:882826 upload attempt=8 succeeded after retry +1752074475 INFO scheduler tick pending=766 claimed=5 completed_last_min=151 +1752074488 DEBUG worker=w10 pool stats idle=14 active=4 waiting=0 +1752074499 INFO worker=w04 job=export:882659 upload attempt=4 succeeded after retry +1752074514 INFO scheduler tick pending=2808 claimed=29 completed_last_min=240 +1752074525 DEBUG worker=w07 pool stats idle=31 active=3 waiting=0 +1752074538 WARN scheduler queue depth 5458 exceeds soft limit 5000 +1752074553 INFO worker=w08 heartbeat ok inflight=4 claimed_total=322 +1752074564 INFO worker=w05 heartbeat ok inflight=20 claimed_total=94 +1752074576 DEBUG worker=w02 pool stats idle=15 active=9 waiting=0 +1752074592 DEBUG worker=w09 pool stats idle=19 active=10 waiting=0 +1752074602 INFO worker=w01 job=export:882046 chunk=7/12 flushed bytes=222511007 +1752074615 DEBUG worker=w09 pool stats idle=4 active=9 waiting=0 +1752074627 INFO worker=w10 job=export:883125 rows=696130 bytes=138647554 dur_ms=1649 state=complete +1752074644 INFO scheduler tick pending=4916 claimed=28 completed_last_min=309 +1752074653 INFO worker=w07 job=export:882154 chunk=4/12 flushed bytes=98634161 +1752074669 WARN scheduler queue depth 5047 exceeds soft limit 5000 +1752074683 DEBUG worker=w02 pool stats idle=16 active=4 waiting=0 +1752074693 INFO worker=w08 job=export:882239 chunk=1/12 flushed bytes=39935077 +1752074705 DEBUG worker=w03 pool stats idle=26 active=2 waiting=0 +1752074720 INFO worker=w13 job=export:884314 rows=775358 bytes=11391441 dur_ms=2556 state=complete +1752074733 INFO scheduler tick pending=4104 claimed=1 completed_last_min=174 +1752074744 INFO worker=w11 job=export:882784 rows=27956 bytes=52628224 dur_ms=7002 state=complete +1752074760 INFO worker=w02 job=export:882519 rows=235734 bytes=309558364 dur_ms=5132 state=complete +``` + + +## Cluster triage session 14 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +webhook-relay-7a518ac89-h5yng 1/1 Running 2 79h 10.42.9.101 ip-10-42-15-133.us-west-2.compute.internal +orders-api-eb43ed05c-p6h35 1/1 Running 1 96h 10.42.5.67 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-4712a5fff-fccyp 0/1 CrashLoopBackOff 14 34m 10.42.26.240 ip-10-42-2-206.us-west-2.compute.internal +webhook-relay-a8f70b44b-pawge 0/1 CrashLoopBackOff 7 8m 10.42.12.137 ip-10-42-2-206.us-west-2.compute.internal +export-scheduler-661fe585f-tsty9 1/1 Running 2 15h 10.42.31.105 ip-10-42-1-169.us-west-2.compute.internal +webhook-relay-b621706c7-ht52r 1/1 Running 3 29m 10.42.26.114 ip-10-42-3-96.us-west-2.compute.internal +auth-gateway-68115fbf2-dek8p 1/1 Running 1 32h 10.42.19.154 ip-10-42-15-133.us-west-2.compute.internal +billing-worker-22290a999-7bud6 0/1 CrashLoopBackOff 7 7m 10.42.27.155 ip-10-42-13-183.us-west-2.compute.internal +rate-limiter-65cf2192b-9mqgm 1/1 Running 0 67h 10.42.26.110 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-9c63ee786-uvdfs 1/1 Running 14 45m 10.42.23.144 ip-10-42-27-195.us-west-2.compute.internal +ledger-sync-befad839c-shtjn 1/1 Running 0 4h 10.42.25.67 ip-10-42-1-169.us-west-2.compute.internal +ledger-sync-3191dcb6c-prtn2 1/1 Running 0 81h 10.42.21.60 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-754405070-ny6z8 1/1 Running 7 15m 10.42.20.66 ip-10-42-16-244.us-west-2.compute.internal +export-scheduler-0d68b027d-z22jc 1/1 Running 1 60h 10.42.0.140 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-70bf1ba81-uu4ha 1/1 Running 14 9m 10.42.23.46 ip-10-42-1-169.us-west-2.compute.internal +billing-worker-d3e2c2d50-bxrmu 1/1 Running 0 3h 10.42.20.200 ip-10-42-2-206.us-west-2.compute.internal +export-scheduler-03de2b4c1-75pta 1/1 Running 0 37h 10.42.0.83 ip-10-42-27-195.us-west-2.compute.internal +search-indexer-371427cf3-rgmu5 1/1 Running 2 36h 10.42.13.181 ip-10-42-12-115.us-west-2.compute.internal +orders-api-25c559909-zbaf7 0/1 CrashLoopBackOff 14 21m 10.42.18.50 ip-10-42-2-206.us-west-2.compute.internal +search-indexer-2d18f7fa1-8yqwt 1/1 Running 14 5m 10.42.28.230 ip-10-42-12-115.us-west-2.compute.internal +rate-limiter-03207a6d1-h4fn7 1/1 Running 7 41m 10.42.6.7 ip-10-42-1-169.us-west-2.compute.internal +ledger-sync-aff29ace0-g9smw 0/1 CrashLoopBackOff 7 24m 10.42.23.207 ip-10-42-13-183.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-12-115.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752010403 INFO [ingest-2] dropping oversized record bytes=87744 topic=events.raw +1752010411 WARN [ingest-0] compaction pass complete segments=39 reclaimed_mb=1073 +1752010415 INFO [ingest-0] compaction pass complete segments=15 reclaimed_mb=601 +1752010425 WARN [ingest-7] compaction pass complete segments=61 reclaimed_mb=2933 +1752010428 INFO [ingest-4] dropping oversized record bytes=18008 topic=events.raw +1752010437 INFO [ingest-1] dropping oversized record bytes=46419 topic=events.raw +1752010442 INFO [ingest-6] rebalance triggered generation=3 members=12 +1752010449 WARN [ingest-4] compaction pass complete segments=4 reclaimed_mb=2261 +1752010456 WARN [ingest-6] compaction pass complete segments=10 reclaimed_mb=1173 +1752010467 INFO [ingest-1] retry attempt=9 for shard=21 after connection reset +1752010471 INFO [ingest-3] rebalance triggered generation=1 members=48 +1752010478 INFO [ingest-6] slow consumer detected partition=31 lag=1461069 +1752010485 INFO [ingest-0] rebalance triggered generation=9 members=6 +1752010491 ERROR [ingest-1] gc pause exceeded budget pause_ms=336 heap_mb=2455 +1752010499 INFO [ingest-4] gc pause exceeded budget pause_ms=1675 heap_mb=1412 +1752010510 INFO [ingest-4] dropping oversized record bytes=55206 topic=events.raw +1752010514 ERROR [ingest-2] rebalance triggered generation=3 members=15 +1752010523 INFO [ingest-6] slow consumer detected partition=3 lag=300789 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2488 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 14: the flaky `checkout-flow` integration job + +**priya** (24:00): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (24:06): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (24:11): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (24:12): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (24:16): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (24:20): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (24:27): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (24:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (24:34): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (24:39): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (24:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (24:46): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16462939091 11m34s 1h +completed checkout-flow integration main push 16465842304 4m31s 3h +completed checkout-flow integration main push 16473076596 6m14s 5h +completed checkout-flow integration main push 16490280192 11m50s 7h +completed checkout-flow integration main push 16466993868 4m47s 9h +completed checkout-flow integration main push 16489447880 10m12s 11h +completed checkout-flow integration main push 16409419931 4m14s 13h +failure checkout-flow integration main push 16466391851 11m13s 15h +completed checkout-flow integration main push 16459917563 4m22s 17h +failure checkout-flow integration main push 16464595644 4m35s 19h +completed checkout-flow integration main push 16452627130 4m57s 21h +completed checkout-flow integration main push 16410352080 10m56s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 26 passed, 23 total +Tests: 1 failed, 186 passed, 195 total +Time: 219.705 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v14: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v13 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 14 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 14 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 14 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3398: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-494: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-506: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-628: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-785: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 14: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752080800 ERROR worker=w04 job=export:884877 upload attempt=4 failed: connection reset by peer, will retry +1752080813 WARN scheduler queue depth 1183 exceeds soft limit 5000 +1752080831 INFO scheduler tick pending=5340 claimed=4 completed_last_min=366 +1752080844 WARN scheduler queue depth 948 exceeds soft limit 5000 +1752080854 WARN scheduler queue depth 3589 exceeds soft limit 5000 +1752080867 INFO scheduler tick pending=7081 claimed=11 completed_last_min=302 +1752080879 INFO scheduler tick pending=4703 claimed=11 completed_last_min=55 +1752080895 INFO worker=w03 job=export:883323 rows=935872 bytes=368652864 dur_ms=9077 state=complete +1752080908 INFO worker=w02 job=export:882287 rows=546082 bytes=280834209 dur_ms=6376 state=complete +1752080919 ERROR worker=w04 job=export:882702 upload attempt=2 failed: connection reset by peer, will retry +1752080935 INFO worker=w01 job=export:881436 state=claimed lease_ms=30000 +1752080943 INFO worker=w03 job=export:884460 rows=231456 bytes=199983705 dur_ms=1508 state=complete +1752080959 WARN worker=w09 job=export:881700 lease renewal took 8574ms (budget 5000ms) +1752080973 INFO worker=w14 heartbeat ok inflight=26 claimed_total=374 +1752080985 WARN worker=w12 job=export:882979 lease renewal took 1158ms (budget 5000ms) +1752080999 WARN worker=w09 job=export:882159 lease renewal took 2433ms (budget 5000ms) +1752081011 INFO worker=w02 job=export:884503 upload attempt=4 succeeded after retry +1752081021 ERROR worker=w14 job=export:881255 upload attempt=2 failed: connection reset by peer, will retry +1752081035 WARN scheduler queue depth 6566 exceeds soft limit 5000 +1752081048 INFO worker=w09 job=export:882255 chunk=6/12 flushed bytes=263097849 +1752081063 INFO worker=w10 job=export:881187 upload attempt=12 succeeded after retry +1752081076 WARN scheduler queue depth 7405 exceeds soft limit 5000 +1752081086 INFO worker=w10 heartbeat ok inflight=22 claimed_total=89 +1752081099 INFO worker=w14 job=export:884543 chunk=8/12 flushed bytes=239321839 +1752081113 INFO worker=w15 job=export:883498 chunk=11/12 flushed bytes=348454856 +1752081127 INFO scheduler tick pending=7848 claimed=15 completed_last_min=294 +1752081143 INFO worker=w16 job=export:884211 chunk=7/12 flushed bytes=78892024 +1752081154 INFO worker=w16 job=export:882611 chunk=6/12 flushed bytes=349897349 +1752081168 INFO scheduler tick pending=3542 claimed=29 completed_last_min=350 +1752081180 INFO worker=w13 job=export:881552 upload attempt=12 succeeded after retry +1752081192 WARN worker=w01 job=export:884697 lease renewal took 9140ms (budget 5000ms) +1752081207 INFO worker=w14 job=export:882593 upload attempt=2 succeeded after retry +1752081218 DEBUG worker=w12 pool stats idle=20 active=6 waiting=0 +1752081229 WARN worker=w14 job=export:882362 lease renewal took 9410ms (budget 5000ms) +1752081246 INFO worker=w15 job=export:883524 chunk=1/12 flushed bytes=82607723 +1752081255 INFO worker=w04 heartbeat ok inflight=11 claimed_total=267 +1752081272 INFO worker=w07 job=export:881114 upload attempt=5 succeeded after retry +1752081283 ERROR worker=w01 job=export:881971 upload attempt=6 failed: connection reset by peer, will retry +1752081299 INFO worker=w09 job=export:884142 chunk=2/12 flushed bytes=164216735 +1752081310 INFO worker=w08 job=export:881866 upload attempt=8 succeeded after retry +1752081324 INFO worker=w16 job=export:881995 rows=382823 bytes=61084025 dur_ms=4712 state=complete +1752081337 INFO worker=w07 job=export:882287 state=claimed lease_ms=30000 +1752081349 INFO worker=w06 job=export:884251 chunk=11/12 flushed bytes=297121337 +1752081364 DEBUG worker=w09 pool stats idle=10 active=1 waiting=0 +1752081372 WARN worker=w07 job=export:882644 lease renewal took 3856ms (budget 5000ms) +1752081385 INFO scheduler tick pending=4124 claimed=13 completed_last_min=29 +1752081402 WARN worker=w13 job=export:881997 lease renewal took 1903ms (budget 5000ms) +1752081415 DEBUG worker=w01 pool stats idle=19 active=12 waiting=0 +1752081427 INFO scheduler tick pending=1432 claimed=12 completed_last_min=301 +1752081442 INFO worker=w15 heartbeat ok inflight=2 claimed_total=59 +1752081453 WARN scheduler queue depth 4442 exceeds soft limit 5000 +1752081467 INFO scheduler tick pending=8164 claimed=32 completed_last_min=81 +1752081478 INFO worker=w16 job=export:881408 state=claimed lease_ms=30000 +1752081491 DEBUG worker=w11 pool stats idle=16 active=6 waiting=0 +1752081505 WARN worker=w07 job=export:884709 lease renewal took 8698ms (budget 5000ms) +1752081517 INFO worker=w13 heartbeat ok inflight=25 claimed_total=230 +1752081528 DEBUG worker=w13 pool stats idle=12 active=12 waiting=0 +1752081545 INFO worker=w15 job=export:883857 chunk=6/12 flushed bytes=3584635 +1752081558 INFO worker=w06 job=export:883365 upload attempt=7 succeeded after retry +1752081567 INFO worker=w15 job=export:882421 chunk=9/12 flushed bytes=204043673 +1752081581 WARN scheduler queue depth 7825 exceeds soft limit 5000 +1752081594 INFO worker=w09 job=export:884688 state=claimed lease_ms=30000 +1752081610 WARN scheduler queue depth 5595 exceeds soft limit 5000 +1752081624 INFO worker=w08 job=export:881832 chunk=11/12 flushed bytes=317542311 +1752081632 INFO worker=w06 job=export:883128 state=claimed lease_ms=30000 +1752081648 DEBUG worker=w12 pool stats idle=9 active=11 waiting=0 +1752081662 INFO scheduler tick pending=7121 claimed=12 completed_last_min=276 +1752081675 INFO worker=w02 job=export:882996 state=claimed lease_ms=30000 +1752081686 DEBUG worker=w07 pool stats idle=15 active=6 waiting=0 +1752081698 INFO worker=w02 job=export:884508 state=claimed lease_ms=30000 +1752081712 INFO scheduler tick pending=2971 claimed=23 completed_last_min=116 +1752081724 INFO worker=w06 job=export:883819 state=claimed lease_ms=30000 +1752081738 ERROR worker=w07 job=export:882216 upload attempt=1 failed: connection reset by peer, will retry +1752081750 WARN worker=w16 job=export:882108 lease renewal took 273ms (budget 5000ms) +1752081767 DEBUG worker=w05 pool stats idle=29 active=8 waiting=0 +1752081780 WARN worker=w09 job=export:883321 lease renewal took 5261ms (budget 5000ms) +1752081789 ERROR worker=w11 job=export:884148 upload attempt=8 failed: connection reset by peer, will retry +1752081801 INFO worker=w03 job=export:883393 chunk=4/12 flushed bytes=244348704 +1752081816 INFO worker=w08 job=export:881229 rows=616389 bytes=329244092 dur_ms=1073 state=complete +1752081832 INFO worker=w01 heartbeat ok inflight=30 claimed_total=122 +1752081840 INFO worker=w03 job=export:882420 state=claimed lease_ms=30000 +1752081858 ERROR worker=w02 job=export:883848 upload attempt=1 failed: connection reset by peer, will retry +1752081869 WARN worker=w03 job=export:882753 lease renewal took 6298ms (budget 5000ms) +1752081884 INFO worker=w16 job=export:882509 upload attempt=7 succeeded after retry +1752081894 WARN worker=w06 job=export:882150 lease renewal took 6706ms (budget 5000ms) +1752081905 WARN worker=w13 job=export:881461 lease renewal took 1579ms (budget 5000ms) +1752081922 INFO worker=w04 job=export:883765 rows=510641 bytes=333908919 dur_ms=1707 state=complete +1752081934 WARN worker=w02 job=export:882550 lease renewal took 9498ms (budget 5000ms) +1752081948 ERROR worker=w01 job=export:881668 upload attempt=9 failed: connection reset by peer, will retry +1752081959 INFO scheduler tick pending=2475 claimed=14 completed_last_min=293 +``` + + +## Cluster triage session 15 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +ledger-sync-d263770a0-z78x4 1/1 Running 2 19h 10.42.9.155 ip-10-42-13-183.us-west-2.compute.internal +search-indexer-def14385c-bj8zx 1/1 Running 2 33h 10.42.26.85 ip-10-42-15-133.us-west-2.compute.internal +rate-limiter-fed64847f-v65ag 1/1 Running 2 72h 10.42.4.35 ip-10-42-13-183.us-west-2.compute.internal +webhook-relay-f9230362a-km6hn 1/1 Running 7 6m 10.42.26.105 ip-10-42-3-96.us-west-2.compute.internal +notify-dispatch-3927cc72c-x9w8h 1/1 Running 0 89h 10.42.31.122 ip-10-42-2-111.us-west-2.compute.internal +notify-dispatch-6678e1e6e-85x3r 1/1 Running 0 61h 10.42.28.64 ip-10-42-16-244.us-west-2.compute.internal +search-indexer-645b4c3aa-tv4fg 1/1 Running 7 34m 10.42.3.115 ip-10-42-27-195.us-west-2.compute.internal +webhook-relay-9fb43c731-dr658 1/1 Running 2 75h 10.42.2.18 ip-10-42-13-183.us-west-2.compute.internal +billing-worker-635b33d72-crsvz 1/1 Running 0 96h 10.42.7.26 ip-10-42-1-169.us-west-2.compute.internal +billing-worker-664d3c79e-d9x7h 0/1 CrashLoopBackOff 7 29m 10.42.24.120 ip-10-42-13-183.us-west-2.compute.internal +webhook-relay-060a70d91-p6zd9 1/1 Running 2 51h 10.42.5.76 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-5f1f18149-gpt5h 1/1 Running 7 11m 10.42.28.141 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-e0f37c8b0-ye6br 0/1 CrashLoopBackOff 7 17m 10.42.31.56 ip-10-42-16-244.us-west-2.compute.internal +billing-worker-714476e82-fsv6b 1/1 Running 0 32h 10.42.26.141 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-ccef5a441-xj4n7 1/1 Running 7 31m 10.42.2.49 ip-10-42-1-169.us-west-2.compute.internal +search-indexer-0afa5468b-4p8em 1/1 Running 2 93h 10.42.13.109 ip-10-42-15-133.us-west-2.compute.internal +rate-limiter-5dd73e367-fxf43 1/1 Running 3 49m 10.42.31.107 ip-10-42-12-115.us-west-2.compute.internal +metrics-ingest-36a83ba5a-uv3fw 1/1 Running 2 90h 10.42.26.75 ip-10-42-3-96.us-west-2.compute.internal +export-scheduler-9fc97c3ec-9xxpk 1/1 Running 14 4m 10.42.16.75 ip-10-42-27-195.us-west-2.compute.internal +metrics-ingest-1710e75b1-qazw8 0/1 CrashLoopBackOff 14 34m 10.42.4.5 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-32921a6d2-3w6qk 1/1 Running 0 50h 10.42.17.201 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-d54005816-zk4zw 1/1 Running 0 74h 10.42.23.82 ip-10-42-3-96.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-12-115.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752014005 INFO [ingest-4] gc pause exceeded budget pause_ms=320 heap_mb=422 +1752014007 INFO [ingest-6] slow consumer detected partition=17 lag=2978775 +1752014018 WARN [ingest-6] checkpoint written offset=2606527 epoch=1 +1752014023 INFO [ingest-3] slow consumer detected partition=11 lag=404832 +1752014032 WARN [ingest-2] dropping oversized record bytes=87456 topic=events.raw +1752014036 INFO [ingest-7] retry attempt=8 for shard=6 after connection reset +1752014046 INFO [ingest-1] slow consumer detected partition=5 lag=3491466 +1752014049 INFO [ingest-0] gc pause exceeded budget pause_ms=1897 heap_mb=2567 +1752014058 INFO [ingest-1] slow consumer detected partition=6 lag=2407203 +1752014063 WARN [ingest-2] dropping oversized record bytes=72857 topic=events.raw +1752014072 INFO [ingest-2] rebalance triggered generation=1 members=53 +1752014077 INFO [ingest-7] slow consumer detected partition=2 lag=2119844 +1752014089 INFO [ingest-4] flush batch size=4096 dur_ms=1573 backlog=61545 +1752014096 INFO [ingest-2] compaction pass complete segments=11 reclaimed_mb=3479 +1752014102 WARN [ingest-6] dropping oversized record bytes=54642 topic=events.raw +1752014105 WARN [ingest-1] checkpoint written offset=1456404 epoch=7 +1752014113 INFO [ingest-1] checkpoint written offset=3398737 epoch=7 +1752014119 INFO [ingest-3] checkpoint written offset=2352065 epoch=3 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2484 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 15: the flaky `checkout-flow` integration job + +**priya** (25:02): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (25:06): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (25:11): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (25:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (25:16): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (25:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (25:26): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (25:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (25:33): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (25:39): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (25:42): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (25:45): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16403429814 7m56s 1h +completed checkout-flow integration main push 16425823765 10m17s 3h +completed checkout-flow integration main push 16449058362 4m26s 5h +completed checkout-flow integration main push 16422928453 11m28s 7h +completed checkout-flow integration main push 16424872191 8m29s 9h +completed checkout-flow integration main push 16482862130 8m23s 11h +completed checkout-flow integration main push 16497790868 4m39s 13h +completed checkout-flow integration main push 16453038431 11m22s 15h +completed checkout-flow integration main push 16471757797 4m45s 17h +completed checkout-flow integration main push 16492616527 9m33s 19h +completed checkout-flow integration main push 16417397499 9m30s 21h +failure checkout-flow integration main push 16450089286 11m12s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 22 passed, 26 total +Tests: 1 failed, 222 passed, 188 total +Time: 227.468 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v15: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v14 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 15 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 15 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 15 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3405: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-465: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-574: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-690: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-727: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 15: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752088004 WARN scheduler queue depth 1334 exceeds soft limit 5000 +1752088014 ERROR worker=w15 job=export:884156 upload attempt=1 failed: connection reset by peer, will retry +1752088028 INFO worker=w15 heartbeat ok inflight=3 claimed_total=173 +1752088042 WARN scheduler queue depth 7519 exceeds soft limit 5000 +1752088057 WARN worker=w05 job=export:884314 lease renewal took 4134ms (budget 5000ms) +1752088067 WARN scheduler queue depth 4908 exceeds soft limit 5000 +1752088083 INFO worker=w12 job=export:883634 upload attempt=8 succeeded after retry +1752088093 DEBUG worker=w05 pool stats idle=26 active=5 waiting=0 +1752088109 INFO worker=w04 job=export:882023 upload attempt=7 succeeded after retry +1752088122 INFO scheduler tick pending=5583 claimed=18 completed_last_min=330 +1752088133 INFO worker=w05 job=export:883065 upload attempt=11 succeeded after retry +1752088145 INFO worker=w04 heartbeat ok inflight=19 claimed_total=58 +1752088160 INFO scheduler tick pending=2911 claimed=5 completed_last_min=371 +1752088171 INFO worker=w03 job=export:881745 upload attempt=12 succeeded after retry +1752088183 INFO worker=w12 job=export:881431 upload attempt=8 succeeded after retry +1752088195 INFO worker=w01 job=export:881851 chunk=3/12 flushed bytes=258574729 +1752088210 INFO worker=w06 heartbeat ok inflight=27 claimed_total=25 +1752088222 INFO worker=w15 job=export:884447 upload attempt=9 succeeded after retry +1752088239 INFO worker=w05 job=export:884354 state=claimed lease_ms=30000 +1752088249 DEBUG worker=w03 pool stats idle=29 active=1 waiting=0 +1752088262 INFO worker=w02 heartbeat ok inflight=29 claimed_total=281 +1752088273 INFO worker=w08 job=export:883097 state=claimed lease_ms=30000 +1752088286 INFO worker=w12 heartbeat ok inflight=15 claimed_total=108 +1752088301 INFO worker=w16 job=export:884067 upload attempt=4 succeeded after retry +1752088313 WARN worker=w06 job=export:883117 lease renewal took 834ms (budget 5000ms) +1752088330 DEBUG worker=w05 pool stats idle=29 active=9 waiting=0 +1752088343 DEBUG worker=w09 pool stats idle=13 active=8 waiting=0 +1752088351 INFO worker=w16 job=export:882556 chunk=1/12 flushed bytes=250420034 +1752088366 INFO worker=w03 job=export:881219 rows=613731 bytes=219909099 dur_ms=6753 state=complete +1752088380 INFO worker=w01 job=export:883039 rows=677916 bytes=262359489 dur_ms=3372 state=complete +1752088391 INFO worker=w01 job=export:883849 state=claimed lease_ms=30000 +1752088404 INFO worker=w03 job=export:882046 chunk=8/12 flushed bytes=242921165 +1752088418 INFO worker=w11 job=export:881972 state=claimed lease_ms=30000 +1752088434 INFO worker=w15 heartbeat ok inflight=25 claimed_total=240 +1752088446 WARN scheduler queue depth 7217 exceeds soft limit 5000 +1752088457 INFO worker=w14 job=export:883369 rows=17948 bytes=227207052 dur_ms=4940 state=complete +1752088470 ERROR worker=w08 job=export:883042 upload attempt=5 failed: connection reset by peer, will retry +1752088481 INFO worker=w06 job=export:882989 rows=285688 bytes=38100145 dur_ms=3450 state=complete +1752088497 INFO scheduler tick pending=8162 claimed=27 completed_last_min=145 +1752088510 ERROR worker=w06 job=export:882271 upload attempt=11 failed: connection reset by peer, will retry +1752088521 INFO worker=w12 job=export:881644 state=claimed lease_ms=30000 +1752088534 DEBUG worker=w10 pool stats idle=17 active=7 waiting=0 +1752088550 INFO worker=w11 heartbeat ok inflight=31 claimed_total=355 +1752088562 WARN scheduler queue depth 2686 exceeds soft limit 5000 +1752088575 INFO worker=w05 job=export:884571 state=claimed lease_ms=30000 +1752088587 INFO worker=w12 job=export:881103 rows=393387 bytes=122393109 dur_ms=5579 state=complete +1752088600 WARN worker=w12 job=export:883502 lease renewal took 8043ms (budget 5000ms) +1752088611 INFO scheduler tick pending=6945 claimed=11 completed_last_min=272 +1752088628 INFO worker=w03 job=export:882879 chunk=11/12 flushed bytes=10615282 +1752088639 INFO worker=w08 heartbeat ok inflight=7 claimed_total=271 +1752088655 WARN scheduler queue depth 2888 exceeds soft limit 5000 +1752088664 INFO worker=w02 job=export:884855 state=claimed lease_ms=30000 +1752088677 WARN scheduler queue depth 3303 exceeds soft limit 5000 +1752088694 INFO worker=w07 job=export:881811 rows=892043 bytes=12022345 dur_ms=2265 state=complete +1752088703 INFO worker=w01 heartbeat ok inflight=15 claimed_total=387 +1752088718 WARN scheduler queue depth 6162 exceeds soft limit 5000 +1752088730 INFO worker=w01 job=export:881170 rows=660821 bytes=358540 dur_ms=7930 state=complete +1752088743 INFO scheduler tick pending=7914 claimed=18 completed_last_min=11 +1752088756 INFO worker=w02 job=export:882709 rows=558652 bytes=355587899 dur_ms=5569 state=complete +1752088771 ERROR worker=w07 job=export:883637 upload attempt=3 failed: connection reset by peer, will retry +1752088783 ERROR worker=w11 job=export:883183 upload attempt=3 failed: connection reset by peer, will retry +1752088795 INFO worker=w01 job=export:883552 state=claimed lease_ms=30000 +1752088807 WARN scheduler queue depth 7593 exceeds soft limit 5000 +1752088819 WARN scheduler queue depth 7428 exceeds soft limit 5000 +1752088834 INFO worker=w11 job=export:884093 state=claimed lease_ms=30000 +1752088848 WARN scheduler queue depth 6204 exceeds soft limit 5000 +1752088859 DEBUG worker=w01 pool stats idle=14 active=3 waiting=0 +1752088872 INFO worker=w05 job=export:883305 state=claimed lease_ms=30000 +1752088886 INFO scheduler tick pending=2920 claimed=18 completed_last_min=81 +1752088899 INFO worker=w12 heartbeat ok inflight=32 claimed_total=250 +1752088912 INFO worker=w10 heartbeat ok inflight=8 claimed_total=340 +1752088928 WARN worker=w15 job=export:882748 lease renewal took 5886ms (budget 5000ms) +1752088940 ERROR worker=w02 job=export:883232 upload attempt=1 failed: connection reset by peer, will retry +1752088953 INFO worker=w06 job=export:883532 upload attempt=4 succeeded after retry +1752088965 WARN worker=w06 job=export:882524 lease renewal took 7344ms (budget 5000ms) +1752088980 WARN worker=w05 job=export:883612 lease renewal took 1745ms (budget 5000ms) +1752088988 INFO worker=w08 job=export:883813 rows=331146 bytes=190625083 dur_ms=7572 state=complete +1752089004 ERROR worker=w15 job=export:882687 upload attempt=11 failed: connection reset by peer, will retry +1752089018 ERROR worker=w12 job=export:881370 upload attempt=11 failed: connection reset by peer, will retry +1752089028 INFO worker=w05 heartbeat ok inflight=15 claimed_total=356 +1752089045 INFO worker=w10 heartbeat ok inflight=6 claimed_total=247 +1752089054 INFO worker=w04 heartbeat ok inflight=28 claimed_total=28 +1752089069 INFO worker=w15 job=export:881766 chunk=9/12 flushed bytes=35100923 +1752089084 INFO scheduler tick pending=4237 claimed=20 completed_last_min=212 +1752089092 ERROR worker=w02 job=export:884754 upload attempt=10 failed: connection reset by peer, will retry +1752089107 INFO worker=w12 heartbeat ok inflight=9 claimed_total=283 +1752089120 INFO scheduler tick pending=5002 claimed=21 completed_last_min=359 +1752089131 INFO worker=w06 job=export:883010 rows=461067 bytes=60854358 dur_ms=4260 state=complete +1752089145 WARN worker=w04 job=export:883286 lease renewal took 3383ms (budget 5000ms) +1752089158 DEBUG worker=w10 pool stats idle=27 active=2 waiting=0 +``` + + +## Cluster triage session 16 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +webhook-relay-dac7e5133-f7vr6 1/1 Running 2 55h 10.42.13.115 ip-10-42-2-111.us-west-2.compute.internal +orders-api-bcfe4a5ac-9r42e 1/1 Running 0 41h 10.42.18.53 ip-10-42-27-195.us-west-2.compute.internal +orders-api-3e536c9b4-zqqs9 1/1 Running 0 25h 10.42.28.84 ip-10-42-13-183.us-west-2.compute.internal +webhook-relay-d4159c044-3y395 0/1 CrashLoopBackOff 14 54m 10.42.0.147 ip-10-42-2-111.us-west-2.compute.internal +rate-limiter-defd138f6-z6fd4 1/1 Running 3 33m 10.42.26.145 ip-10-42-15-133.us-west-2.compute.internal +orders-api-f849ce4b5-9m8wf 0/1 CrashLoopBackOff 7 47m 10.42.17.165 ip-10-42-2-206.us-west-2.compute.internal +webhook-relay-d92304ae3-tfpzc 1/1 Running 0 13h 10.42.22.215 ip-10-42-1-169.us-west-2.compute.internal +billing-worker-f95e28e63-yvmca 1/1 Running 2 3h 10.42.22.247 ip-10-42-15-133.us-west-2.compute.internal +metrics-ingest-d9d82c341-jt8pk 0/1 CrashLoopBackOff 14 4m 10.42.29.79 ip-10-42-2-111.us-west-2.compute.internal +orders-api-71ae2d545-mmx27 1/1 Running 14 47m 10.42.15.75 ip-10-42-2-111.us-west-2.compute.internal +orders-api-a788a59db-v9344 1/1 Running 0 62h 10.42.26.128 ip-10-42-12-115.us-west-2.compute.internal +orders-api-e53dc0686-u2db9 1/1 Running 1 28h 10.42.16.127 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-f59132a50-hfa2c 1/1 Running 2 6h 10.42.1.72 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-660564bcd-t2wmn 0/1 CrashLoopBackOff 14 51m 10.42.22.131 ip-10-42-13-183.us-west-2.compute.internal +auth-gateway-563cdaac6-j2dhh 1/1 Running 3 20m 10.42.30.107 ip-10-42-15-133.us-west-2.compute.internal +search-indexer-431c0a725-q84hr 1/1 Running 0 8h 10.42.31.188 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-d2b45760a-akpm7 1/1 Running 2 6h 10.42.20.181 ip-10-42-3-96.us-west-2.compute.internal +webhook-relay-b695a0593-jtfns 1/1 Running 2 96h 10.42.26.13 ip-10-42-27-195.us-west-2.compute.internal +export-scheduler-4a9d43e1d-nhaxe 1/1 Running 0 91h 10.42.10.220 ip-10-42-2-111.us-west-2.compute.internal +metrics-ingest-b88a617b6-ewx8z 1/1 Running 7 22m 10.42.3.213 ip-10-42-2-111.us-west-2.compute.internal +ledger-sync-e7418f929-8ufjh 1/1 Running 1 41h 10.42.30.182 ip-10-42-2-206.us-west-2.compute.internal +billing-worker-d49dd8b72-8y8su 1/1 Running 0 20h 10.42.10.241 ip-10-42-2-111.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-12-115.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752017605 WARN [ingest-5] retry attempt=6 for shard=18 after connection reset +1752017610 INFO [ingest-0] gc pause exceeded budget pause_ms=220 heap_mb=2390 +1752017616 WARN [ingest-0] flush batch size=4096 dur_ms=446 backlog=64585 +1752017623 INFO [ingest-2] compaction pass complete segments=56 reclaimed_mb=981 +1752017630 INFO [ingest-1] flush batch size=4096 dur_ms=277 backlog=30775 +1752017638 INFO [ingest-6] dropping oversized record bytes=53678 topic=events.raw +1752017647 INFO [ingest-3] gc pause exceeded budget pause_ms=1603 heap_mb=2177 +1752017650 INFO [ingest-4] checkpoint written offset=249116 epoch=2 +1752017658 INFO [ingest-5] dropping oversized record bytes=62551 topic=events.raw +1752017665 INFO [ingest-6] flush batch size=4096 dur_ms=2228 backlog=7724 +1752017672 INFO [ingest-7] flush batch size=4096 dur_ms=644 backlog=34857 +1752017678 INFO [ingest-2] compaction pass complete segments=62 reclaimed_mb=2846 +1752017688 INFO [ingest-7] flush batch size=4096 dur_ms=470 backlog=49851 +1752017691 INFO [ingest-1] dropping oversized record bytes=79795 topic=events.raw +1752017701 INFO [ingest-6] flush batch size=4096 dur_ms=380 backlog=49614 +1752017709 INFO [ingest-2] gc pause exceeded budget pause_ms=956 heap_mb=1340 +1752017713 ERROR [ingest-6] checkpoint written offset=228586 epoch=6 +1752017723 INFO [ingest-2] checkpoint written offset=2438744 epoch=8 +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2870 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 16: the flaky `checkout-flow` integration job + +**priya** (26:01): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (26:04): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (26:11): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (26:13): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (26:19): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (26:20): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (26:26): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (26:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (26:34): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (26:39): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (26:41): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (26:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +failure checkout-flow integration main push 16447698066 5m59s 1h +failure checkout-flow integration main push 16402202272 9m54s 3h +completed checkout-flow integration main push 16402055509 6m59s 5h +completed checkout-flow integration main push 16463009297 8m36s 7h +failure checkout-flow integration main push 16448469391 7m26s 9h +completed checkout-flow integration main push 16420322816 6m35s 11h +failure checkout-flow integration main push 16426423551 10m46s 13h +completed checkout-flow integration main push 16413031157 11m40s 15h +completed checkout-flow integration main push 16401841225 6m26s 17h +completed checkout-flow integration main push 16462586653 8m55s 19h +completed checkout-flow integration main push 16444905214 11m40s 21h +completed checkout-flow integration main push 16423937730 6m57s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 24 passed, 25 total +Tests: 1 failed, 230 passed, 236 total +Time: 209.680 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v16: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v15 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 16 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 16 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 16 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3412: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-453: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-503: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-600: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-799: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 16: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752095202 INFO worker=w14 job=export:882041 upload attempt=3 succeeded after retry +1752095216 WARN worker=w13 job=export:881821 lease renewal took 4180ms (budget 5000ms) +1752095230 ERROR worker=w02 job=export:882565 upload attempt=9 failed: connection reset by peer, will retry +1752095241 INFO worker=w03 job=export:882521 rows=752212 bytes=296797907 dur_ms=6558 state=complete +1752095253 INFO worker=w11 heartbeat ok inflight=22 claimed_total=395 +1752095267 INFO worker=w02 job=export:884011 rows=755902 bytes=344331462 dur_ms=8894 state=complete +1752095282 INFO scheduler tick pending=6925 claimed=3 completed_last_min=280 +1752095292 INFO worker=w01 heartbeat ok inflight=16 claimed_total=162 +1752095308 INFO worker=w13 job=export:883218 chunk=8/12 flushed bytes=134333090 +1752095322 ERROR worker=w09 job=export:881561 upload attempt=11 failed: connection reset by peer, will retry +1752095331 WARN worker=w11 job=export:881020 lease renewal took 5755ms (budget 5000ms) +1752095347 WARN scheduler queue depth 2548 exceeds soft limit 5000 +1752095357 DEBUG worker=w05 pool stats idle=3 active=12 waiting=0 +1752095370 WARN scheduler queue depth 4666 exceeds soft limit 5000 +1752095384 INFO worker=w03 job=export:884951 rows=736102 bytes=267093853 dur_ms=7020 state=complete +1752095400 WARN scheduler queue depth 6170 exceeds soft limit 5000 +1752095413 INFO worker=w06 heartbeat ok inflight=32 claimed_total=92 +1752095425 INFO worker=w05 job=export:884850 state=claimed lease_ms=30000 +1752095436 INFO worker=w10 job=export:881982 rows=795873 bytes=13778537 dur_ms=6642 state=complete +1752095447 INFO scheduler tick pending=4687 claimed=14 completed_last_min=210 +1752095461 WARN worker=w09 job=export:882818 lease renewal took 4109ms (budget 5000ms) +1752095476 INFO worker=w06 job=export:883103 rows=255319 bytes=214520388 dur_ms=1861 state=complete +1752095486 DEBUG worker=w16 pool stats idle=28 active=3 waiting=0 +1752095499 WARN scheduler queue depth 5775 exceeds soft limit 5000 +1752095517 ERROR worker=w10 job=export:882931 upload attempt=2 failed: connection reset by peer, will retry +1752095526 INFO worker=w10 job=export:882077 upload attempt=10 succeeded after retry +1752095542 DEBUG worker=w06 pool stats idle=6 active=5 waiting=0 +1752095551 INFO worker=w13 job=export:882964 rows=944630 bytes=10784713 dur_ms=6859 state=complete +1752095569 INFO scheduler tick pending=6470 claimed=28 completed_last_min=47 +1752095582 INFO scheduler tick pending=1653 claimed=26 completed_last_min=35 +1752095593 INFO worker=w11 heartbeat ok inflight=16 claimed_total=84 +1752095603 INFO scheduler tick pending=5315 claimed=3 completed_last_min=330 +1752095621 INFO worker=w12 job=export:882616 chunk=7/12 flushed bytes=121627301 +1752095634 WARN scheduler queue depth 6985 exceeds soft limit 5000 +1752095646 INFO worker=w15 job=export:882102 state=claimed lease_ms=30000 +1752095660 WARN scheduler queue depth 1081 exceeds soft limit 5000 +1752095673 INFO scheduler tick pending=4362 claimed=28 completed_last_min=55 +1752095685 DEBUG worker=w13 pool stats idle=10 active=1 waiting=0 +1752095697 WARN worker=w15 job=export:884601 lease renewal took 5209ms (budget 5000ms) +1752095712 ERROR worker=w06 job=export:882775 upload attempt=5 failed: connection reset by peer, will retry +1752095720 INFO worker=w14 job=export:881063 chunk=11/12 flushed bytes=230433418 +1752095733 INFO worker=w01 job=export:881274 state=claimed lease_ms=30000 +1752095746 WARN worker=w11 job=export:882760 lease renewal took 6022ms (budget 5000ms) +1752095764 INFO scheduler tick pending=2835 claimed=25 completed_last_min=211 +1752095776 INFO worker=w02 job=export:884786 upload attempt=1 succeeded after retry +1752095790 INFO worker=w13 job=export:882228 state=claimed lease_ms=30000 +1752095803 WARN scheduler queue depth 7192 exceeds soft limit 5000 +1752095811 DEBUG worker=w01 pool stats idle=19 active=2 waiting=0 +1752095826 WARN worker=w16 job=export:883350 lease renewal took 2813ms (budget 5000ms) +1752095841 INFO worker=w02 job=export:882650 upload attempt=11 succeeded after retry +1752095850 INFO scheduler tick pending=4955 claimed=18 completed_last_min=344 +1752095868 INFO worker=w02 job=export:883586 chunk=2/12 flushed bytes=186308324 +1752095878 INFO worker=w07 job=export:881173 chunk=4/12 flushed bytes=90209462 +1752095893 INFO worker=w12 job=export:882825 state=claimed lease_ms=30000 +1752095902 INFO worker=w07 job=export:884195 upload attempt=11 succeeded after retry +1752095919 WARN worker=w08 job=export:881481 lease renewal took 8133ms (budget 5000ms) +1752095931 INFO worker=w10 job=export:883474 state=claimed lease_ms=30000 +1752095944 DEBUG worker=w09 pool stats idle=7 active=12 waiting=0 +1752095959 INFO worker=w06 job=export:884182 rows=482424 bytes=320616894 dur_ms=2161 state=complete +1752095969 INFO worker=w01 job=export:882526 state=claimed lease_ms=30000 +1752095983 INFO scheduler tick pending=786 claimed=14 completed_last_min=180 +1752095995 INFO worker=w03 job=export:883458 chunk=4/12 flushed bytes=39460831 +1752096011 INFO worker=w13 heartbeat ok inflight=14 claimed_total=326 +1752096020 INFO worker=w13 job=export:881998 upload attempt=1 succeeded after retry +1752096032 WARN worker=w05 job=export:883410 lease renewal took 6415ms (budget 5000ms) +1752096045 DEBUG worker=w11 pool stats idle=7 active=10 waiting=0 +1752096063 INFO worker=w15 job=export:881152 chunk=5/12 flushed bytes=119365969 +1752096072 DEBUG worker=w02 pool stats idle=25 active=11 waiting=0 +1752096087 INFO worker=w04 job=export:881111 state=claimed lease_ms=30000 +1752096101 DEBUG worker=w01 pool stats idle=1 active=11 waiting=0 +1752096111 DEBUG worker=w06 pool stats idle=7 active=12 waiting=0 +1752096125 INFO scheduler tick pending=4830 claimed=6 completed_last_min=98 +1752096136 INFO scheduler tick pending=4349 claimed=23 completed_last_min=364 +1752096151 INFO scheduler tick pending=7134 claimed=31 completed_last_min=241 +1752096166 WARN worker=w16 job=export:882445 lease renewal took 2336ms (budget 5000ms) +1752096178 INFO worker=w01 job=export:883399 state=claimed lease_ms=30000 +1752096190 INFO worker=w01 job=export:884219 rows=682920 bytes=376236482 dur_ms=7880 state=complete +1752096202 INFO worker=w10 job=export:881866 rows=488458 bytes=227331824 dur_ms=315 state=complete +1752096215 INFO worker=w06 job=export:883954 state=claimed lease_ms=30000 +1752096230 DEBUG worker=w05 pool stats idle=6 active=5 waiting=0 +1752096244 INFO worker=w01 job=export:884551 chunk=6/12 flushed bytes=291785843 +1752096257 WARN scheduler queue depth 5867 exceeds soft limit 5000 +1752096269 WARN worker=w06 job=export:884694 lease renewal took 7188ms (budget 5000ms) +1752096283 WARN scheduler queue depth 5346 exceeds soft limit 5000 +1752096293 WARN worker=w12 job=export:882915 lease renewal took 6602ms (budget 5000ms) +1752096306 INFO worker=w13 job=export:883680 rows=896583 bytes=170162163 dur_ms=9072 state=complete +1752096321 INFO worker=w05 job=export:884539 chunk=6/12 flushed bytes=96279202 +1752096331 INFO worker=w06 job=export:884173 upload attempt=5 succeeded after retry +1752096345 INFO scheduler tick pending=2513 claimed=13 completed_last_min=383 +1752096362 INFO worker=w09 job=export:881843 state=claimed lease_ms=30000 +``` + + +## Cluster triage session 17 — pod restarts on the ingest tier + +Dana asked why the metrics-ingest deployment kept cycling after the 14:05 rollout. Pulled the pod list first to see how widespread the restarts were, since the alert only fired for one availability zone and we were not sure whether the node pool autoscaler had anything to do with it. The suspicion at this point was a bad readiness probe timeout introduced in the previous chart bump, but nothing was confirmed yet and the on-call notes from last week mentioned a similar pattern that turned out to be a noisy neighbor on the shared node group. + +``` +$ kubectl get pods -n platform -o wide --sort-by=.status.startTime +NAME READY STATUS RESTARTS AGE IP NODE +webhook-relay-104af7e59-49t32 1/1 Running 2 82h 10.42.24.174 ip-10-42-12-115.us-west-2.compute.internal +notify-dispatch-f70c16a89-xudef 1/1 Running 14 18m 10.42.14.59 ip-10-42-2-206.us-west-2.compute.internal +metrics-ingest-424c3f8ce-2bdzg 1/1 Running 1 2h 10.42.5.161 ip-10-42-16-244.us-west-2.compute.internal +rate-limiter-97d74be77-cus74 1/1 Running 1 41h 10.42.19.123 ip-10-42-2-206.us-west-2.compute.internal +search-indexer-1139bb2e9-7vvz6 1/1 Running 0 78h 10.42.28.146 ip-10-42-15-133.us-west-2.compute.internal +metrics-ingest-88535b8c1-9mafj 1/1 Running 7 48m 10.42.26.3 ip-10-42-27-195.us-west-2.compute.internal +billing-worker-5fac967b8-87kku 1/1 Running 0 92h 10.42.16.127 ip-10-42-15-133.us-west-2.compute.internal +ledger-sync-dcce9df52-hvkrz 1/1 Running 3 4m 10.42.6.123 ip-10-42-2-111.us-west-2.compute.internal +rate-limiter-1ee2b6f82-6j99k 1/1 Running 2 74h 10.42.6.236 ip-10-42-13-183.us-west-2.compute.internal +webhook-relay-cf590b04f-zamc6 1/1 Running 0 25h 10.42.26.24 ip-10-42-27-195.us-west-2.compute.internal +rate-limiter-ce1cab5b3-jqnxd 1/1 Running 3 23m 10.42.0.46 ip-10-42-2-111.us-west-2.compute.internal +webhook-relay-eef298b75-8cc33 1/1 Running 0 85h 10.42.25.199 ip-10-42-1-169.us-west-2.compute.internal +export-scheduler-bb7bde3c6-ymnnx 1/1 Running 3 48m 10.42.29.23 ip-10-42-27-195.us-west-2.compute.internal +export-scheduler-34f952113-ra39j 0/1 CrashLoopBackOff 7 22m 10.42.30.204 ip-10-42-12-115.us-west-2.compute.internal +billing-worker-6fca11cbf-5yruj 1/1 Running 0 23h 10.42.6.94 ip-10-42-2-206.us-west-2.compute.internal +orders-api-38e63947b-qd6u8 0/1 CrashLoopBackOff 14 43m 10.42.18.88 ip-10-42-15-133.us-west-2.compute.internal +metrics-ingest-b24cf6129-8styd 0/1 CrashLoopBackOff 14 49m 10.42.5.119 ip-10-42-15-133.us-west-2.compute.internal +orders-api-a600d90df-za7sc 0/1 CrashLoopBackOff 14 38m 10.42.4.109 ip-10-42-16-244.us-west-2.compute.internal +webhook-relay-47a53c5ab-2jzw3 1/1 Running 1 15h 10.42.4.142 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-cdf940203-t87pb 0/1 CrashLoopBackOff 14 24m 10.42.8.79 ip-10-42-27-195.us-west-2.compute.internal +auth-gateway-5df659825-ng5cn 1/1 Running 0 63h 10.42.8.126 ip-10-42-16-244.us-west-2.compute.internal +metrics-ingest-abb5e9608-ykza2 1/1 Running 2 25h 10.42.2.236 ip-10-42-2-111.us-west-2.compute.internal +``` + +``` +$ kubectl describe pod metrics-ingest-7d9f4c8b6-x2m4q -n platform | tail -n 26 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 41m default-scheduler Successfully assigned platform/metrics-ingest-7d9f4c8b6-x2m4q to ip-10-42-2-111.us-west-2.compute.internal + Normal Pulling 41m kubelet Pulling image "registry.internal/platform/metrics-ingest:2026.27.3" + Normal Pulled 40m kubelet Successfully pulled image in 12.402s + Normal Created 40m kubelet Created container metrics-ingest + Normal Started 40m kubelet Started container metrics-ingest + Warning Unhealthy 38m (x3 over 39m) kubelet Readiness probe failed: Get "http://10.42.7.114:8081/healthz": context deadline exceeded + Warning Unhealthy 37m (x2 over 38m) kubelet Liveness probe failed: HTTP probe failed with statuscode: 503 + Normal Killing 37m kubelet Container metrics-ingest failed liveness probe, will be restarted + Warning BackOff 12m (x41 over 35m) kubelet Back-off restarting failed container +``` + +The probe failures line up with a GC pause spike in the container logs, so the next step was to grab a heap profile before the pod got killed again. The tricky part is that the profiler endpoint is only enabled when the pod starts with PROFILING=1, and flipping that env var means another rollout, which resets the very state we are trying to observe. Jordan suggested attaching an ephemeral debug container instead, which worked on the second attempt after we remembered the cluster still runs the older admission policy that blocks ephemeral containers without a namespace label. + +``` +$ kubectl logs metrics-ingest-7d9f4c8b6-x2m4q -n platform --previous | tail -n 18 +1752021201 WARN [ingest-5] dropping oversized record bytes=53461 topic=events.raw +1752021209 WARN [ingest-2] gc pause exceeded budget pause_ms=1183 heap_mb=2566 +1752021215 INFO [ingest-6] compaction pass complete segments=22 reclaimed_mb=3475 +1752021223 WARN [ingest-4] compaction pass complete segments=44 reclaimed_mb=825 +1752021233 ERROR [ingest-6] gc pause exceeded budget pause_ms=894 heap_mb=1554 +1752021240 WARN [ingest-3] dropping oversized record bytes=26963 topic=events.raw +1752021243 WARN [ingest-6] gc pause exceeded budget pause_ms=1477 heap_mb=1507 +1752021254 INFO [ingest-0] slow consumer detected partition=24 lag=1734194 +1752021260 ERROR [ingest-4] rebalance triggered generation=1 members=33 +1752021268 INFO [ingest-6] checkpoint written offset=418663 epoch=8 +1752021273 ERROR [ingest-2] retry attempt=2 for shard=3 after connection reset +1752021282 INFO [ingest-7] retry attempt=3 for shard=27 after connection reset +1752021285 INFO [ingest-4] checkpoint written offset=986636 epoch=5 +1752021296 WARN [ingest-1] rebalance triggered generation=5 members=46 +1752021302 ERROR [ingest-6] retry attempt=1 for shard=5 after connection reset +1752021309 INFO [ingest-3] dropping oversized record bytes=20194 topic=events.raw +1752021312 WARN [ingest-6] flush batch size=4096 dur_ms=1941 backlog=49058 +1752021323 WARN [ingest-3] dropping oversized record bytes=48732 topic=events.raw +``` + +Outcome for this pass: bumped the readiness timeout from 2s to 5s in the values override and pinned the deployment to the c6i node group while the heap issue gets a proper fix. Filed PLAT-2433 to track the allocation regression, and left a note in the runbook that the ephemeral-container workaround needs the `debug-ok` namespace label. Restarts stopped after the change rolled out, though nobody is fully convinced the timeout was the root cause rather than a symptom of the slow consumer path doing synchronous work on the health check goroutine, which Ravi wants to refactor next sprint anyway. + + +## Debugging thread 17: the flaky `checkout-flow` integration job + +**priya** (27:01): the job failed again on main, third time this week, always the same spec but a different assertion each run + +**marcus** (27:05): I reran with --repeat 50 locally and could not reproduce once, which usually means it is timing dependent on the shared runner + +**kim** (27:08): the runner class changed two weeks ago from m5.large to m5.xlarge, so if anything it got faster, and faster is exactly when these races show up + +**noel** (27:12): pulled the junit artifacts from the last six red runs and diffed them; the failure is always inside the polling helper, never the assertion itself + +**ravi** (27:16): the polling helper caps at 2000ms with a 50ms interval, and the container cold-start on the new runners eats about 1400ms before the server even binds + +**priya** (27:22): so the fix might just be to gate the suite on a readiness ping instead of a fixed sleep, which we should have done from the start + +**marcus** (27:26): I tried that on a branch and the flake rate dropped from roughly 8% to zero across 200 runs, opening a PR + +**kim** (27:31): before we merge, can we also delete the retry-on-red step from the workflow? it has been masking this for months and skews the duration metrics + +**noel** (27:34): agreed, retries hide real regressions; I will remove it and add a comment explaining why so nobody re-adds it in a panic + +**ravi** (27:36): one more thing: the fixture database snapshot is rebuilt on every run and takes 90 seconds; caching it keyed on the migrations hash saves most of that + +**priya** (27:40): cache key needs to include the seed script too, we got burned by that in the exports suite last quarter + +**marcus** (27:44): merged; watching the next 20 runs on main before closing the ticket, will post the flake dashboard link here + +``` +$ gh run list --workflow integration.yml --limit 12 +STATUS TITLE BRANCH EVENT ID ELAPSED AGE +completed checkout-flow integration main push 16480851213 4m56s 1h +failure checkout-flow integration main push 16411365057 4m52s 3h +completed checkout-flow integration main push 16446437723 6m46s 5h +completed checkout-flow integration main push 16477175802 7m21s 7h +completed checkout-flow integration main push 16410647721 7m28s 9h +completed checkout-flow integration main push 16432277502 5m19s 11h +completed checkout-flow integration main push 16424333979 9m58s 13h +completed checkout-flow integration main push 16427101706 8m35s 15h +completed checkout-flow integration main push 16439208755 7m13s 17h +completed checkout-flow integration main push 16405697054 11m23s 19h +completed checkout-flow integration main push 16415830259 11m31s 21h +completed checkout-flow integration main push 16426801958 6m27s 23h +``` + +``` +FAIL test/integration/checkout_flow.spec.ts (14.02 s) + checkout flow + ✓ creates a draft order from the cart (812 ms) + ✓ applies a percentage promotion to eligible lines (233 ms) + ✗ finalizes payment intent within the polling window (2044 ms) + Timeout: condition not met within 2000ms + at pollUntil (test/helpers/poll.ts:31:11) + at Object. (test/integration/checkout_flow.spec.ts:118:5) + ✓ emits an order.confirmed event exactly once (154 ms) + +Test Suites: 1 failed, 23 passed, 25 total +Tests: 1 failed, 196 passed, 208 total +Time: 217.663 s +``` + +Retrospective note added to the testing guide: any helper that waits for an external condition must derive its budget from an explicit readiness signal, not a constant. Constants encode assumptions about hardware that quietly rot. The 200-run soak on the fix branch is the strongest evidence we have collected for a flake fix so far, and the team agreed to require a soak like it for any future change that claims to fix nondeterminism, since the alternative — merging on vibes and watching main — has cost us roughly a day of aggregate engineer attention per week this quarter. + + +## Database migration plan v17: splitting `events` into hot and archive tiers + +Context: the `events` table is 2.1 TB and 96% of reads touch rows newer than 30 days. Vacuum is taking 11 hours, index bloat on `(account_id, created_at)` is at 38%, and the nightly export job now overlaps with morning peak in Europe. The plan below is the third revision after review comments from the storage group; the main change since v16 is doing the backfill in keyset-paginated batches rather than by ctid ranges, because ctid ranges break when autovacuum relocates tuples mid-copy and we saw exactly that in staging. + +| Phase | Action | Est. duration | Rollback | Owner | +|---|---|---|---|---| +| 1 | Create `events_hot` partitioned by week, identical columns, no FKs yet | 20 min | drop table | elena | +| 2 | Dual-write via trigger on `events` insert path, monitored for drift | 3 days soak | disable trigger | elena | +| 3 | Backfill last 45 days in 50k-row keyset batches, throttled to 4k rows/s | ~14 h | truncate `events_hot` | ravi | +| 4 | Verify counts + checksums per day-bucket, alert on any mismatch > 0 | 2 h | n/a (read only) | kim | +| 5 | Flip read path behind `events_hot_reads` flag at 1% → 25% → 100% | 2 days | flag to 0% | marcus | +| 6 | Repoint export job and retention worker to the new table | 1 h | revert config | noel | +| 7 | Rename old table to `events_archive`, revoke app-role writes | 10 min | rename back | elena | +| 8 | Detach-and-drop archive partitions older than 400 days, per legal hold list | rolling | restore from snapshot | noel | + +```sql +-- Phase 3 backfill batch, keyset pagination on (created_at, id) +INSERT INTO events_hot (id, account_id, kind, payload, created_at) +SELECT id, account_id, kind, payload, created_at +FROM events +WHERE (created_at, id) > ($1, $2) + AND created_at >= now() - interval '45 days' +ORDER BY created_at, id +LIMIT 50000; +``` + +Open questions from review, with current thinking: + +- Trigger overhead during dual-write measured at 4.1% p99 latency on the insert path in staging; acceptable, but we will watch the payment-adjacent writers specifically because their SLO headroom is thinnest. +- The drift monitor compares per-minute counts between tables; it needs to tolerate the replication delay window or it will page on every deploy. Proposal: compare minute N only after minute N+2 closes. +- Legal hold list lives in a spreadsheet today. Phase 8 will not run until it is a table with an audit trail, full stop — this was the one hard blocker from the review. +- Do we keep the trigger permanently as a safety net? Consensus: no, remove it two weeks after phase 7, because permanent triggers become invisible load-bearing infrastructure that nobody remembers exists. +- Autovacuum settings for the new partitioned table should start at the cluster default and only be tuned with evidence; the old table accumulated seven layers of bespoke settings that nobody can explain anymore. + +Rehearsal results from the staging run on the 2 TB anonymized snapshot: the backfill sustained 3.7k rows/s under throttle, checksum verification found zero mismatches across 45 day-buckets, and the read-path flag flip showed no latency regression at any percentage step. The one surprise was WAL volume — the dual-write phase roughly doubles it, and the archiver briefly fell behind, so production gets a temporary bump to the WAL sender budget for the soak window plus an alert if archive lag exceeds five minutes. Elena wants the whole plan rehearsed once more after the keyset change, which is scheduled for Thursday night. + + +## Dependency upgrade review, batch 17 + +Quarterly pass through the lockfile. Rules of engagement as usual: security advisories first, then majors with migration guides, then the long tail of minors in one batch PR per workspace. Anything touching serialization or auth gets its own PR with a soak. Notes per package follow. + +### fastify 4.28.1 → 5.3.2 (major) + +Route shorthand for HEAD changed; our health endpoints declared both GET and HEAD explicitly so we hit the duplicate-route error at boot. Fix was deleting the redundant HEAD declarations. Also the logger option no longer accepts a bare boolean in the same way; wrapped in the new config object. Bench shows ~6% throughput gain on the echo route, which is nice but not why we upgraded. + +### pg 8.11.5 → 8.13.1 (minor) + +Picked up the fix for the double-release crash on pool timeout that we have a workaround for in ledger-sync; removing the workaround is a separate PR so the diff stays legible. Changelog also mentions SCRAM iteration count changes — no action, our servers already advertise the higher count. + +### zod 3.23.8 → 3.24.0 (minor) + +No behavior change for us, but the new error map API deprecates the one we monkey-patched. Filed a chore to migrate before it becomes a major-version blocker. Grepped for the deprecated call: 14 sites, all in the request validation layer, mechanical change. + +### undici 6.19.2 → 6.21.0 (minor) + +Advisory GHSA-c76h fixed here (header smuggling under a proxy config we do not use, but the scanner flags it regardless). Drop-in. + +### vitest 1.6.0 → 2.1.9 (major) + +Snapshot format changed; 212 snapshots rewrote on first run. Verified a sample of 30 by eye, the rest by the fact that the underlying serializers produce equivalent structures. The pool option rename (threads → forks default) actually fixed our lingering worker-leak warning. + +### aws-sdk client-s3 3.550.0 → 3.700.0 (minor batch) + +Chunked the diff review by release notes; nothing behavioral for our call sites (GetObject, PutObject, multipart). The checksum-by-default change lands in a future major, noted in the tracking issue. + +### luxon 3.4.4 → 3.5.0 (minor) + +Fixes the Etc/GMT offset parsing edge we documented in INV-118. Our regression test from that incident passes against the new version without the workaround branch, so the workaround gets deleted. + +### eslint 8.57.0 → 9.14.0 (major) + +Flat config migration. This is the big one: 40 minutes of mechanical translation plus two plugins that needed version bumps of their own to be compatible. The old .eslintrc files are deleted, not left as dead config, per the batch-1 lesson. + +``` +$ pnpm audit --prod +┌─────────────────────┬────────────────────────────────────────────────┐ +│ severity │ 0 critical, 0 high, 1 moderate, 3 low │ +│ moderate │ transitive via legacy-archiver (dev-only path) │ +│ action │ tracked in SEC-441, upstream fix unreleased │ +└─────────────────────┴────────────────────────────────────────────────┘ +``` + +Batch outcome: 61 packages bumped across 4 PRs, all green after the vitest snapshot churn settled. The eslint migration surfaced 9 previously-masked lint errors, of which 2 were real bugs (an unawaited promise in the export scheduler and a shadowed variable in the retry helper) — a decent return on an otherwise tedious chore, and the strongest argument yet for not letting the linter drift three majors behind again. + + +## Quarterly planning notes — session 17 + +Attendees: dana (facilitating), marcus, priya, elena, ravi, kim, noel, sofia. Async pre-reads were the capacity model, the reliability review, and the support ticket taxonomy from last quarter. Notes are paraphrased, decisions bolded. + +### Capacity + +The team enters the quarter at 7.5 engineer-equivalents after accounting for on-call rotation, interviews, and Sofia's parental leave starting week 6. Last quarter we planned to 92% of capacity and landed at 71% delivered, so this quarter plans to 75% with an explicit slack pool for interrupts. **Decision: commit to three initiatives, not five.** + +### Reliability review + +Error budget spend was dominated by the two ingest incidents; both trace back to unbounded queue growth under partial broker failure. The proposed fix (backpressure at the producer with shed-and-alert semantics) is the top engineering initiative. **Decision: backpressure work is P0 and staffed with two people, not one, because the last two single-staffed reliability projects both stalled at the review stage.** + +### Support taxonomy + +38% of tickets last quarter were export-related, and of those, most were 'where is my file' rather than actual failures. The export status surface is the fix, not more support macros. **Decision: export status page ships this quarter; success metric is export ticket volume halved by week 10.** + +### Deferred + +The multi-account switcher, the audit-log search rewrite, and the sandbox environment refresh are explicitly deferred with names attached to the deferral so the next planning session knows who to ask. Deferring without an owner for the deferral is how things get silently dropped. + +### Risks + +The broker upgrade forced by the EOL notice lands mid-quarter and is sized at two weeks but has historically-poor estimate accuracy (last one took five). It is scheduled first, not last, so the tail risk lands on the slack pool instead of on the committed work. + +### Hiring + +One backend req approved, targeting a start before week 8 to overlap with Sofia. Interview loop load is capped at 3 hours/person/week and counted against capacity rather than pretended to be free. + +Action items: + +- [ ] marcus: write the backpressure design one-pager by Friday, circulate before the deep-dive +- [ ] priya: instrument export ticket tagging so the week-10 metric has a baseline before the work starts +- [ ] elena: confirm broker upgrade window with the platform team and book the rehearsal slot +- [ ] kim: convert the capacity model spreadsheet into the shared dashboard so it stops living in a DM +- [ ] dana: publish the deferral list with owners in the team space and link it from the quarter doc +- [ ] noel: schedule the mid-quarter checkpoint for week 6, before the leave starts, not after + + +## API pagination redesign — discussion round 17 + +The public list endpoints still use offset pagination and it is now a real problem: page 4000 of the orders list does a 2-second scan, integrators poll deep pages on a schedule, and rows shifting between pages during writes causes the duplicate-and-missing-items class of bug reports we keep re-triaging. This round of discussion is about committing to cursor pagination and the deprecation path, not about whether — that was settled last round. + +Proposed contract: + +```json +{ + "data": [ + { + "id": "ord_8fk2m1", + "status": "confirmed", + "total_cents": 45900, + "created_at": "2026-07-01T18:22:05Z" + } + ], + "page_info": { + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMVQxODoyMjowNVoiLCJpZCI6Im9yZF84ZmsybTEifQ", + "page_size": 100 + } +} +``` + +- The cursor encodes the keyset tuple (created_at, id), opaque and base64url. Opaque matters: the previous 'cursor' attempt leaked a raw offset inside and integrators started constructing their own, which is why that deprecation failed. +- Cursors are valid for 24 hours, enforced with an embedded expiry, so we retain the freedom to change the encoding without a versioned migration. Expired cursors return a 400 with a specific error code, not a generic one, so client libraries can distinguish 'restart the walk' from 'you sent garbage'. +- Sort options are constrained to indexed keysets: created_at (default) and updated_at. The long tail of arbitrary sort params on the offset API — some of which trigger filesorts — gets no cursor equivalent, and the four integrators who use them get direct outreach rather than a surprise. +- page_size caps at 500, up from 100, because half the deep-paging traffic is integrators working around the small page size. Bigger pages plus cursors should eliminate most of the pathological access pattern on its own. +- The offset params keep working on the old paths for 12 months with a Sunset header and a per-key deprecation dashboard, because the last deprecation taught us that emails get ignored but a dashboard the key owner can see gets acted on. +- Backfill-style consumers who genuinely want 'everything' get pointed at the bulk export endpoint instead; pagination is the wrong tool for full-table sync no matter how it is implemented, and saying so explicitly in the docs prevents the next generation of workarounds. + +``` +$ curl -s 'https://api.example.internal/v2/orders?page_size=2' | jq .page_info +{ + "has_next": true, + "next_cursor": "eyJjcmVhdGVkX2F0IjoiMjAyNi0wNy0wMlQwOToxNDo1NVoiLCJpZCI6Im9yZF85cWsxbXoifQ", + "page_size": 2 +} +``` + +Remaining disagreement, recorded rather than resolved: whether `has_previous` and backward cursors ship in v2.0 or v2.1. Ravi argues that bidirectional paging doubles the index requirements and no integrator has asked for it; Kim argues that adding it later changes the cursor encoding and burns the one clean migration we get. Current lean is ship-forward-only and reserve an encoding version byte in the cursor so a later addition is non-breaking. Decision deadline is the API review on the 19th, and whoever feels strongest writes the one-pager. + + +## Incident review INC-3419: webhook delivery backlog + +Duration 94 minutes, customer impact: delayed (not lost) webhook delivery for roughly 12% of endpoints. No data loss. This review follows the blameless template; the timeline is reconstructed from the pager, the deploy log, and the dispatcher's own metrics, which disagreed with each other in one interesting place noted below. + +| Time (UTC) | Event | +|---|---| +| 13:02 | Deploy of notify-dispatch 2026.27.1 begins, canary healthy | +| 13:11 | Full rollout complete; delivery success rate nominal | +| 13:47 | p99 delivery latency begins climbing; no alert (threshold set on success rate, not latency) | +| 14:09 | First customer report via support: webhooks arriving 20+ minutes late | +| 14:15 | On-call paged manually by support escalation | +| 14:22 | Backlog identified: 340k pending deliveries, growing at 2k/min | +| 14:31 | Root cause hypothesis: new per-endpoint circuit breaker holds a global lock while evaluating | +| 14:38 | Rollback initiated to 2026.26.4 | +| 14:49 | Rollback complete; backlog begins draining at 9k/min | +| 15:36 | Backlog fully drained; incident closed | + +What made it worse: the dispatcher reports its own queue depth, and that metric flatlined during the event because the reporting goroutine was starved by the same lock. The externally-measured backlog (from the broker side) is what surfaced the truth. Lesson recorded: any self-reported health metric needs an external counterpart, because the failure mode that matters is exactly the one that compromises self-reporting. + +Follow-ups: + +- NOTIF-497: alert on delivery latency p99, not just success rate — done during the review itself +- NOTIF-579: move circuit breaker state to sharded locks; load test at 5x current endpoint count before re-rollout +- NOTIF-686: broker-side queue depth becomes the paging signal; dispatcher-side depth demoted to debugging +- NOTIF-746: canary stage extended to 30 minutes for this service, since the lock contention needed sustained load to manifest + + +## Raw capture 17: export-scheduler worker logs during the backlog drain + +Kept for reference while writing the postmortem; the interesting pattern is the lease-renewal warnings clustering right before each throughput dip. + +``` +1752102400 INFO scheduler tick pending=5201 claimed=15 completed_last_min=23 +1752102417 INFO worker=w14 job=export:884808 state=claimed lease_ms=30000 +1752102426 INFO scheduler tick pending=1449 claimed=21 completed_last_min=336 +1752102444 INFO worker=w14 job=export:882800 rows=164735 bytes=70137505 dur_ms=353 state=complete +1752102453 INFO worker=w11 job=export:882103 state=claimed lease_ms=30000 +1752102468 WARN scheduler queue depth 1973 exceeds soft limit 5000 +1752102483 WARN scheduler queue depth 267 exceeds soft limit 5000 +1752102496 INFO worker=w05 job=export:882116 upload attempt=9 succeeded after retry +1752102509 WARN scheduler queue depth 4472 exceeds soft limit 5000 +1752102519 WARN worker=w04 job=export:883403 lease renewal took 8711ms (budget 5000ms) +1752102532 INFO worker=w12 job=export:882661 chunk=11/12 flushed bytes=86373185 +1752102545 INFO worker=w08 job=export:882410 rows=129690 bytes=142069736 dur_ms=3013 state=complete +1752102558 INFO worker=w07 job=export:881809 rows=870006 bytes=162185099 dur_ms=1867 state=complete +1752102574 ERROR worker=w09 job=export:882547 upload attempt=4 failed: connection reset by peer, will retry +1752102584 INFO scheduler tick pending=6053 claimed=23 completed_last_min=317 +1752102597 INFO worker=w09 job=export:882809 chunk=3/12 flushed bytes=189079916 +1752102611 INFO scheduler tick pending=666 claimed=15 completed_last_min=46 +1752102623 INFO worker=w01 heartbeat ok inflight=3 claimed_total=331 +1752102634 DEBUG worker=w15 pool stats idle=27 active=4 waiting=0 +1752102647 WARN scheduler queue depth 5890 exceeds soft limit 5000 +1752102660 INFO worker=w09 job=export:884708 upload attempt=4 succeeded after retry +1752102677 INFO worker=w09 job=export:882126 upload attempt=7 succeeded after retry +1752102690 INFO worker=w05 heartbeat ok inflight=6 claimed_total=248 +1752102702 INFO worker=w12 job=export:881735 chunk=12/12 flushed bytes=135680593 +1752102717 INFO worker=w09 job=export:882298 rows=911011 bytes=301265086 dur_ms=5103 state=complete +1752102729 DEBUG worker=w03 pool stats idle=28 active=1 waiting=0 +1752102743 WARN scheduler queue depth 4006 exceeds soft limit 5000 +1752102756 WARN worker=w15 job=export:881267 lease renewal took 6873ms (budget 5000ms) +1752102764 WARN worker=w10 job=export:882840 lease renewal took 2020ms (budget 5000ms) +1752102782 INFO worker=w01 job=export:882500 rows=756146 bytes=271124066 dur_ms=4267 state=complete +1752102792 INFO worker=w07 job=export:883852 state=claimed lease_ms=30000 +1752102806 INFO worker=w12 job=export:884511 chunk=6/12 flushed bytes=84123551 +1752102818 INFO worker=w03 job=export:884088 rows=99530 bytes=309304034 dur_ms=5624 state=complete +1752102833 ERROR worker=w11 job=export:883421 upload attempt=4 failed: connection reset by peer, will retry +1752102847 INFO worker=w16 heartbeat ok inflight=31 claimed_total=90 +1752102858 DEBUG worker=w16 pool stats idle=29 active=2 waiting=0 +1752102868 DEBUG worker=w02 pool stats idle=13 active=2 waiting=0 +1752102883 INFO worker=w10 job=export:881725 chunk=9/12 flushed bytes=94990394 +1752102894 ERROR worker=w04 job=export:882785 upload attempt=10 failed: connection reset by peer, will retry +1752102911 INFO worker=w12 heartbeat ok inflight=20 claimed_total=218 +1752102921 INFO worker=w11 job=export:881556 state=claimed lease_ms=30000 +1752102936 INFO worker=w12 job=export:881876 rows=417150 bytes=377183611 dur_ms=4033 state=complete +1752102951 ERROR worker=w16 job=export:883510 upload attempt=2 failed: connection reset by peer, will retry +1752102959 INFO worker=w15 job=export:884179 upload attempt=9 succeeded after retry +1752102972 INFO worker=w11 job=export:882727 chunk=7/12 flushed bytes=240685745 +1752102989 WARN scheduler queue depth 3086 exceeds soft limit 5000 +1752103000 INFO scheduler tick pending=1494 claimed=11 completed_last_min=198 +1752103013 INFO worker=w08 job=export:881157 chunk=3/12 flushed bytes=216020124 +1752103029 INFO worker=w06 heartbeat ok inflight=9 claimed_total=315 +1752103042 INFO worker=w11 job=export:882131 rows=786639 bytes=143798176 dur_ms=4982 state=complete +1752103051 INFO worker=w08 job=export:884380 chunk=12/12 flushed bytes=298630521 +1752103064 ERROR worker=w01 job=export:884092 upload attempt=12 failed: connection reset by peer, will retry +1752103076 DEBUG worker=w01 pool stats idle=23 active=12 waiting=0 +1752103092 WARN worker=w16 job=export:881868 lease renewal took 7960ms (budget 5000ms) +1752103106 INFO worker=w05 job=export:884963 chunk=3/12 flushed bytes=133106975 +1752103115 INFO scheduler tick pending=2909 claimed=16 completed_last_min=389 +1752103128 INFO worker=w16 job=export:884648 upload attempt=7 succeeded after retry +1752103143 WARN worker=w13 job=export:883458 lease renewal took 4289ms (budget 5000ms) +1752103159 INFO worker=w01 heartbeat ok inflight=21 claimed_total=348 +1752103172 INFO worker=w10 job=export:882416 upload attempt=11 succeeded after retry +1752103183 INFO worker=w14 job=export:883313 upload attempt=2 succeeded after retry +1752103193 INFO worker=w06 job=export:882079 chunk=7/12 flushed bytes=317463548 +1752103210 INFO worker=w05 job=export:884688 upload attempt=11 succeeded after retry +1752103224 INFO worker=w10 job=export:881451 chunk=6/12 flushed bytes=190543917 +1752103232 INFO worker=w05 heartbeat ok inflight=24 claimed_total=19 +1752103247 INFO worker=w07 job=export:883090 state=claimed lease_ms=30000 +1752103261 INFO worker=w03 job=export:881674 rows=524492 bytes=59400304 dur_ms=5450 state=complete +1752103273 WARN worker=w08 job=export:883507 lease renewal took 318ms (budget 5000ms) +1752103289 INFO scheduler tick pending=7858 claimed=26 completed_last_min=348 +1752103300 INFO scheduler tick pending=3850 claimed=7 completed_last_min=31 +1752103313 INFO worker=w04 job=export:884064 rows=278670 bytes=278224024 dur_ms=4210 state=complete +1752103325 INFO worker=w16 job=export:883268 state=claimed lease_ms=30000 +1752103336 INFO worker=w12 job=export:884105 rows=37931 bytes=152604976 dur_ms=2143 state=complete +1752103354 INFO worker=w11 heartbeat ok inflight=18 claimed_total=228 +1752103364 INFO worker=w09 heartbeat ok inflight=6 claimed_total=101 +1752103380 INFO worker=w02 heartbeat ok inflight=23 claimed_total=296 +1752103392 INFO worker=w11 job=export:881282 chunk=9/12 flushed bytes=327934502 +1752103406 INFO worker=w15 heartbeat ok inflight=19 claimed_total=237 +1752103419 INFO worker=w14 job=export:884200 state=claimed lease_ms=30000 +1752103428 INFO worker=w06 job=export:884266 state=claimed lease_ms=30000 +1752103444 INFO worker=w16 job=export:883404 chunk=1/12 flushed bytes=164019187 +1752103454 DEBUG worker=w07 pool stats idle=27 active=7 waiting=0 +1752103470 INFO worker=w09 job=export:882549 rows=413057 bytes=229041751 dur_ms=3849 state=complete +1752103479 INFO worker=w03 job=export:884227 upload attempt=5 succeeded after retry +1752103496 INFO worker=w16 job=export:881363 state=claimed lease_ms=30000 +1752103506 INFO scheduler tick pending=5076 claimed=10 completed_last_min=284 +1752103522 INFO worker=w09 job=export:882648 rows=615229 bytes=294236164 dur_ms=2284 state=complete +1752103533 INFO worker=w01 job=export:881056 state=claimed lease_ms=30000 +1752103548 WARN worker=w02 job=export:884938 lease renewal took 5952ms (budget 5000ms) +1752103560 INFO worker=w06 job=export:881499 rows=10305 bytes=237985287 dur_ms=8584 state=complete +``` + diff --git a/packages/vessel-light/demo/eval.config.json b/packages/vessel-light/demo/eval.config.json new file mode 100644 index 00000000..72d30e38 --- /dev/null +++ b/packages/vessel-light/demo/eval.config.json @@ -0,0 +1,21 @@ +{ + "package": "../.ghost", + "asks": "./asks.md", + "ballast": "./ballast.md", + "runsPerCell": 5, + "arms": { + "naked": true, + "dump": true, + "gather": true, + "dump-growth": { + "extraPackages": [ + "../../vessel-react/.ghost", + "../../../apps/docs/.ghost" + ], + "asks": [1] + } + }, + "tells": "./median-tells.json", + "out": "./out", + "ghostBin": "node ../../ghost/dist/bin.js" +} diff --git a/packages/vessel-light/demo/median-score.mjs b/packages/vessel-light/demo/median-score.mjs new file mode 100644 index 00000000..2d01465d --- /dev/null +++ b/packages/vessel-light/demo/median-score.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +// Median score: count measured unsteered-generation tells in an HTML artifact. +// Grounded in the antimedian experiment (300 generations, 3 models, no design +// context). Higher score = more median. Usage: +// node median-score.mjs [more.html…] +// node median-score.mjs out/A/*/*.html +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const { tells } = JSON.parse( + readFileSync(join(here, "median-tells.json"), "utf8"), +); + +export function scoreHtml(html, tells, { discountTells = [] } = {}) { + const discounts = new Set(discountTells); + const hits = []; + let score = 0; + let max = 0; + + for (const tell of tells) { + if (discounts.has(tell.id)) continue; + max += tell.weight; + const count = countTell(html, tell); + if (count > 0) { + hits.push({ id: tell.id, count }); + score += tell.weight; + } + } + + return { score, max, hits }; +} + +function countTell(html, tell) { + if (tell.kind === "text") { + if (!tell.pattern) return 0; + return html.split(tell.pattern).length - 1; + } + + const regex = new RegExp(tell.pattern, "giu"); + let count = 0; + for (const _match of html.matchAll(regex)) count += 1; + return count; +} + +const isScript = process.argv[1] === fileURLToPath(import.meta.url); + +if (isScript) { + const files = process.argv.slice(2); + if (files.length === 0) { + console.error("usage: median-score.mjs [...]"); + process.exit(1); + } + + const rows = []; + for (const file of files) { + const html = readFileSync(file, "utf8"); + const { score, hits } = scoreHtml(html, tells); + rows.push({ file, score, hits: hits.map((hit) => hit.id) }); + } + + const max = tells.reduce((s, t) => s + t.weight, 0); + for (const { file, score, hits } of rows.sort((a, b) => b.score - a.score)) { + console.log(`${String(score).padStart(3)}/${max} ${file}`); + for (const h of hits) console.log(` - ${h}`); + } + if (rows.length > 1) { + const mean = rows.reduce((s, r) => s + r.score, 0) / rows.length; + console.log( + `\nmean: ${mean.toFixed(1)}/${max} across ${rows.length} files`, + ); + } +} diff --git a/packages/vessel-light/demo/median-tells.json b/packages/vessel-light/demo/median-tells.json new file mode 100644 index 00000000..13213370 --- /dev/null +++ b/packages/vessel-light/demo/median-tells.json @@ -0,0 +1,97 @@ +{ + "source": "antimedian (300 unsteered generations, gemini/gpt/opus, 2026-07)", + "note": "counts are antimedian pattern occurrences; weight reflects distinctiveness-destroying power. Sanctioned-by-vessel median patterns (pills, clamp headings, max-width containers, email tables) are deliberately absent.", + "tells": [ + { + "id": "hover-lift-family", + "antimedianCount": 341, + "kind": "regex", + "pattern": ":hover[^}]*transform:[^};]*translateY\\(-", + "weight": 3 + }, + { + "id": "indigo-blue-default-accent", + "antimedianCount": 60, + "kind": "regex", + "pattern": "#(4f46e5|6366f1|4338ca|2563eb|3b82f6|1d4ed8|4a90e2|4361ee)\\b", + "weight": 3 + }, + { + "id": "unprompted-dark-theme", + "antimedianCount": 271, + "kind": "regex", + "pattern": "(?:(?:^|[}\\s,])(?:body|html|:root)\\s*(?:,[^{}]*)?\\{[^}]*background(?:-color)?:\\s*(?:#(?:[0-2][0-9a-fA-F]){3}\\b|#[0-2]{3}\\b|black\\b|rgba?\\(\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d)|--(?:bg|background|surface)[\\w-]*:\\s*(?:#(?:[0-2][0-9a-fA-F]){3}\\b|#[0-2]{3}\\b|black\\b|rgba?\\(\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d\\s*,\\s*[0-4]?\\d))", + "weight": 2 + }, + { + "id": "gradient-background", + "antimedianCount": 63, + "kind": "regex", + "pattern": "background[^;]*:(linear|radial)-gradient|background-image:\\s*(linear|radial)-gradient", + "weight": 3 + }, + { + "id": "glassmorphism", + "antimedianCount": 23, + "kind": "regex", + "pattern": "backdrop-filter:\\s*blur", + "weight": 2 + }, + { + "id": "chat-bubble-avatar", + "antimedianCount": 57, + "kind": "regex", + "pattern": "(avatar[^}]*border-radius:\\s*50%|initials)", + "weight": 1 + }, + { + "id": "emoji-icons", + "antimedianCount": 26, + "kind": "regex", + "pattern": "[\\u{1F300}-\\u{1FAFF}\\u{2600}-\\u{27BF}]", + "weight": 1 + }, + { + "id": "stock-pricing-copy", + "antimedianCount": 14, + "kind": "text", + "pattern": "Simple, transparent pricing", + "weight": 2 + }, + { + "id": "welcome-back-copy", + "antimedianCount": 10, + "kind": "text", + "pattern": "Welcome back", + "weight": 1 + }, + { + "id": "sticky-top-header", + "antimedianCount": 239, + "kind": "regex", + "pattern": "(header|nav)[^}]*position:\\s*(sticky|fixed)[^}]*top:\\s*0", + "weight": 1 + }, + { + "id": "viewport-centered-card", + "antimedianCount": 152, + "kind": "regex", + "pattern": "(body|\\.container)[^}]*display:\\s*flex[^}]*(align-items:\\s*center[^}]*justify-content:\\s*center|justify-content:\\s*center[^}]*align-items:\\s*center)[^}]*(min-)?height:\\s*100(vh|dvh)", + "weight": 1 + }, + { + "id": "inter-font-default", + "antimedianCount": 44, + "kind": "regex", + "pattern": "font-family:[^;]*\\bInter\\b", + "weight": 1 + }, + { + "id": "segoe-font-default", + "antimedianCount": 104, + "kind": "regex", + "pattern": "font-family:[^;]*Segoe UI", + "weight": 1 + } + ] +} diff --git a/packages/vessel-light/demo/run-arm.md b/packages/vessel-light/demo/run-arm.md new file mode 100644 index 00000000..67d057b6 --- /dev/null +++ b/packages/vessel-light/demo/run-arm.md @@ -0,0 +1,31 @@ +# Arm assembly (for the driving agent) + +All arms: fresh context per run. Same model, same temperature. Write output to +out///run-.html. + +## Arm A — naked +1. Load ballast.md. +2. Give the ask. No fingerprint anywhere. + +## Arm B — dump +1. Concatenate every node in packages/vessel-light/.ghost/*.md (bodies, in + filename order) at the TOP of context — the naive system-prompt integration. +2. Load ballast.md after it. +3. Give the ask. Do not mention gather or selection. + +## Arm B-clean (optional) +Same as B without ballast. + +## Arm C — gather +1. Load ballast.md. +2. Run: ghost gather "" --package packages/vessel-light/.ghost +3. Let the agent select nodes against the ask, then: + ghost pull --package packages/vessel-light/.ghost +4. Give the ask. Pulls sit near the ask, after ballast. + +## Scoring +1. Stage each artifact in a scratch git repo containing the .ghost package; + run ghost review --no-probes; record which checks fire and what a reviewing + agent flags per artifact. +2. Screenshot every artifact (agent-browser). Assemble per-ask grids: A | B | C. +3. Report consistency across the 3–5 runs per cell, not single wins. diff --git a/packages/vessel/.ghost/.gitignore b/packages/vessel-react/.ghost/.gitignore similarity index 100% rename from packages/vessel/.ghost/.gitignore rename to packages/vessel-react/.ghost/.gitignore diff --git a/packages/vessel/.ghost/asset.registry.md b/packages/vessel-react/.ghost/asset.registry.md similarity index 91% rename from packages/vessel/.ghost/asset.registry.md rename to packages/vessel-react/.ghost/asset.registry.md index e2a30a2a..7da690f0 100644 --- a/packages/vessel/.ghost/asset.registry.md +++ b/packages/vessel-react/.ghost/asset.registry.md @@ -1,9 +1,9 @@ --- description: The shadcn registry is the distribution surface — copy-and-own, with decision metadata as part of the API. materials: - - packages/vessel/registry.json - - packages/vessel/public/r/** - - packages/vessel/.shadcn/skills.md + - packages/vessel-react/registry.json + - packages/vessel-react/public/r/** + - packages/vessel-react/.shadcn/skills.md --- Vessel is distributed as a generated shadcn registry, not an npm package. diff --git a/packages/vessel/.ghost/asset.tokens.md b/packages/vessel-react/.ghost/asset.tokens.md similarity index 94% rename from packages/vessel/.ghost/asset.tokens.md rename to packages/vessel-react/.ghost/asset.tokens.md index cb0a96bd..302f70f9 100644 --- a/packages/vessel/.ghost/asset.tokens.md +++ b/packages/vessel-react/.ghost/asset.tokens.md @@ -1,8 +1,8 @@ --- description: The token contract — primitives feed semantic roles, extensions stay narrow and job-named, no broad alias sprawl. materials: - - packages/vessel/src/styles/main.css - - packages/vessel/src/styles/*.css + - packages/vessel-react/src/styles/main.css + - packages/vessel-react/src/styles/*.css --- The token contract has a fixed shape that all token work must preserve: diff --git a/packages/vessel/.ghost/checks/escape-hatches-governed.md b/packages/vessel-react/.ghost/checks/escape-hatches-governed.md similarity index 100% rename from packages/vessel/.ghost/checks/escape-hatches-governed.md rename to packages/vessel-react/.ghost/checks/escape-hatches-governed.md diff --git a/packages/vessel/.ghost/checks/reference-stays-agnostic.md b/packages/vessel-react/.ghost/checks/reference-stays-agnostic.md similarity index 100% rename from packages/vessel/.ghost/checks/reference-stays-agnostic.md rename to packages/vessel-react/.ghost/checks/reference-stays-agnostic.md diff --git a/packages/vessel/.ghost/checks/registry-metadata-current.md b/packages/vessel-react/.ghost/checks/registry-metadata-current.md similarity index 100% rename from packages/vessel/.ghost/checks/registry-metadata-current.md rename to packages/vessel-react/.ghost/checks/registry-metadata-current.md diff --git a/packages/vessel/.ghost/checks/token-contract-holds.md b/packages/vessel-react/.ghost/checks/token-contract-holds.md similarity index 100% rename from packages/vessel/.ghost/checks/token-contract-holds.md rename to packages/vessel-react/.ghost/checks/token-contract-holds.md diff --git a/packages/vessel/.ghost/condition.escape-hatches.md b/packages/vessel-react/.ghost/condition.escape-hatches.md similarity index 95% rename from packages/vessel/.ghost/condition.escape-hatches.md rename to packages/vessel-react/.ghost/condition.escape-hatches.md index ee411725..f04710d0 100644 --- a/packages/vessel/.ghost/condition.escape-hatches.md +++ b/packages/vessel-react/.ghost/condition.escape-hatches.md @@ -1,7 +1,7 @@ --- description: When component work needs className, inline style, or arbitrary values — the governed escape-hatch path. materials: - - packages/vessel/scripts/audit-agent-safety.mjs + - packages/vessel-react/scripts/audit-agent-safety.mjs --- Condition: you are writing or reviewing Vessel component code and the change diff --git a/packages/vessel/.ghost/condition.upstream-sync.md b/packages/vessel-react/.ghost/condition.upstream-sync.md similarity index 100% rename from packages/vessel/.ghost/condition.upstream-sync.md rename to packages/vessel-react/.ghost/condition.upstream-sync.md diff --git a/packages/vessel/.ghost/glossary.md b/packages/vessel-react/.ghost/glossary.md similarity index 100% rename from packages/vessel/.ghost/glossary.md rename to packages/vessel-react/.ghost/glossary.md diff --git a/packages/vessel/.ghost/index.md b/packages/vessel-react/.ghost/index.md similarity index 100% rename from packages/vessel/.ghost/index.md rename to packages/vessel-react/.ghost/index.md diff --git a/packages/vessel-react/.ghost/manifest.yml b/packages/vessel-react/.ghost/manifest.yml new file mode 100644 index 00000000..2d2f2f4e --- /dev/null +++ b/packages/vessel-react/.ghost/manifest.yml @@ -0,0 +1,2 @@ +schema: ghost.fingerprint-package/v1 +id: vessel-react diff --git a/packages/vessel/.ghost/principle.named-decisions.md b/packages/vessel-react/.ghost/principle.named-decisions.md similarity index 93% rename from packages/vessel/.ghost/principle.named-decisions.md rename to packages/vessel-react/.ghost/principle.named-decisions.md index 27d73f56..2c2e8340 100644 --- a/packages/vessel/.ghost/principle.named-decisions.md +++ b/packages/vessel-react/.ghost/principle.named-decisions.md @@ -1,8 +1,8 @@ --- description: Make off-system output hard to express — decision names beat raw values, checks beat prose. materials: - - packages/vessel/src/components/** - - packages/vessel/scripts/audit-agent-safety.mjs + - packages/vessel-react/src/components/** + - packages/vessel-react/scripts/audit-agent-safety.mjs --- Vessel's agent-safety discipline: **make off-system output hard to express, diff --git a/packages/vessel/.ghost/principle.reference-not-brand.md b/packages/vessel-react/.ghost/principle.reference-not-brand.md similarity index 100% rename from packages/vessel/.ghost/principle.reference-not-brand.md rename to packages/vessel-react/.ghost/principle.reference-not-brand.md diff --git a/packages/vessel/.shadcn/skills.md b/packages/vessel-react/.shadcn/skills.md similarity index 100% rename from packages/vessel/.shadcn/skills.md rename to packages/vessel-react/.shadcn/skills.md diff --git a/packages/vessel/README.md b/packages/vessel-react/README.md similarity index 95% rename from packages/vessel/README.md rename to packages/vessel-react/README.md index 378f7022..fd3b695b 100644 --- a/packages/vessel/README.md +++ b/packages/vessel-react/README.md @@ -61,7 +61,7 @@ npx shadcn add / Or build the library locally for workspace linking: ```bash -pnpm --filter @design-intelligence/vessel build:lib +pnpm --filter @design-intelligence/vessel-react build:lib ``` See [`apps/docs`](../../apps/docs) for the live component catalogue. @@ -71,8 +71,8 @@ See [`apps/docs`](../../apps/docs) for the live component catalogue. `vessel` also ships a `vessel-mcp` bin — a Model Context Protocol server that re-exposes the component registry to AI assistants (Claude Code, Cursor, etc.) so they can discover and install components without a human in the loop. 5 tools, 2 resources. Source lives in `src/mcp/`, built separately via `tsconfig.mcp.json` → `dist-mcp/`. ```bash -pnpm --filter @design-intelligence/vessel build:mcp -node packages/vessel/dist-mcp/bin.js # stdio server +pnpm --filter @design-intelligence/vessel-react build:mcp +node packages/vessel-react/dist-mcp/bin.js # stdio server ``` Wire it into your MCP host by pointing at the `vessel-mcp` bin. diff --git a/packages/vessel/components.json b/packages/vessel-react/components.json similarity index 100% rename from packages/vessel/components.json rename to packages/vessel-react/components.json diff --git a/packages/vessel-react/fingerprint/.events b/packages/vessel-react/fingerprint/.events new file mode 100644 index 00000000..14a7fa82 --- /dev/null +++ b/packages/vessel-react/fingerprint/.events @@ -0,0 +1,2 @@ +{"ts":"2026-07-08T11:24:21.294Z","event":"gather","menu":["anti-goal.median","contract.theming","contract.tokens","index","pattern.conversation","primitive.composition","primitive.controls"],"wild":false,"wildIds":[]} +{"ts":"2026-07-08T11:24:37.764Z","event":"pull","ids":["primitive.controls"],"inlinedMaterials":11,"omittedMaterials":0} diff --git a/packages/vessel-react/fingerprint/anti-goal.median.md b/packages/vessel-react/fingerprint/anti-goal.median.md new file mode 100644 index 00000000..56569ca9 --- /dev/null +++ b/packages/vessel-react/fingerprint/anti-goal.median.md @@ -0,0 +1,43 @@ +--- +description: "The measured defaults of unsteered generation — gather before any build to know what the model will reach for on its own, and veer." +--- + +These are not aesthetic opinions. They are the measured convergence of 300 +unsteered generations across three frontier models (the antimedian +experiment): the defaults a model reaches for when nobody hands it a stance. +An output showing several of these tells reads as generated, whatever else +it does right. + +Reject hover-lift as the default interaction: cards, buttons, or anything +else rising on `translateY` with a growing shadow. The single most +convergent motion tell (341 combined occurrences). Vessel confirms hover +with color and background shifts, not levitation — the vendored components +already encode this; do not add lift on top. + +Reject the indigo/blue default accent (`#4f46e5`, `#2563eb` and neighbors) — +the most convergent accent choice. Vessel is monochrome with expressive color +confined to the chart roles; an indigo CTA is the model speaking, not a +decision. + +Reject unprompted dark theme. Dark was the second most common unsteered +choice (271). Dark here is the explicit `.dark` theme a user opts into, +never a whole-page default nobody asked for. + +Reject gradient page backgrounds — radial washes, diagonal blue-purple, +gradient-filled CTAs — and glassmorphism (`backdrop-filter: blur` cards). +Model-signature moves. Surfaces are flat semantic roles. + +Reject chat bubbles with initials-circle avatars for assistant turns. The +measured chat median is alternating balloons; the conversation grammar here +is plain assistant prose on the page surface. + +Reject emoji as icons and emoji as imagery. The icon library and text labels +carry meaning. + +Reject stock template copy: "Simple, transparent pricing" appeared verbatim +across all three models. If a heading could be swapped into a competitor's +page unchanged, write a different heading. + +Two median patterns this component set deliberately shares, so their +presence is fidelity here, not drift: pill-shaped controls and fluid clamp +display headings. Convergence is not the crime; surrendering the choice is. diff --git a/packages/vessel-react/fingerprint/checks/component-integrity.md b/packages/vessel-react/fingerprint/checks/component-integrity.md new file mode 100644 index 00000000..cd7505e8 --- /dev/null +++ b/packages/vessel-react/fingerprint/checks/component-integrity.md @@ -0,0 +1,38 @@ +--- +name: Component integrity +description: Flags variant sprawl, one-off forks of vendored components, hierarchy violations, and app truth hardening into the shared UI layer. +severity: medium +references: + - primitive.controls + - contract.tokens + - index +--- + +These flags guard the vendored component set as this repo builds on it. + +Flag more than one primary-variant button per view. Secondary actions step +down to `secondary`, `outline`, `ghost`, or `link`. If two actions genuinely +compete, the hierarchy needs a decision, not two primaries. + +Flag new variants or props added to a vendored component without evidence of +repeated need — a second or third call site wanting the same thing. One +screen's need is a `className` at the call site or a local composition, not +a new variant in the shared set. + +Flag long ad-hoc `className` strings at call sites that reimplement an +existing variant (a hand-rolled outline button, a re-padded card). Recommend +the existing variant, or promoting the pattern if it truly is new. + +Flag copies of vendored components (`button-v2.tsx`, `custom-dialog.tsx`) +that fork instead of composing. Wrappers that compose are fine; parallel +forks split the system. + +Flag product-specific copy, business flows, or app-only assumptions being +hardcoded into the shared `ui/` or `ai-elements/` layer as if universal. +Product truth belongs in app-level components and this repo's own +fingerprint nodes, not in the reference layer. + +Flag interactive elements missing their accessibility contract when the +vendored component provided one: focus-visible rings removed, `aria-invalid` +wiring dropped from forms, dialogs without titles. The contract shipped +working; a diff should not un-ship it. diff --git a/packages/vessel-react/fingerprint/checks/median-tells.md b/packages/vessel-react/fingerprint/checks/median-tells.md new file mode 100644 index 00000000..6fa8b394 --- /dev/null +++ b/packages/vessel-react/fingerprint/checks/median-tells.md @@ -0,0 +1,40 @@ +--- +name: Median tells +description: Flags the measured defaults of unsteered generation — hover-lift, default accents, unprompted dark theme, gradients, chat bubbles, emoji icons, stock copy. +severity: high +references: + - anti-goal.median + - pattern.conversation +--- + +These flags target the measured convergence patterns of unsteered model +generation. Each is mechanically detectable in a diff. + +Flag `translateY` inside a `:hover` rule or a `hover:` utility (for example +`hover:-translate-y-*`) on cards, buttons, or list items, especially paired +with a shadow increase. Hover confirmation here is color and background +change, not lift. + +Flag accent values in the indigo/blue default family (`#4f46e5`, `#6366f1`, +`#2563eb`, `#3b82f6`, and close neighbors, including `indigo-*` and `blue-*` +utilities) unless the diff shows the user asked for them. They are model +defaults, not palette members. + +Flag whole-page dark backgrounds when the ask did not request dark mode. +Dark surfaces here are the explicit `.dark` theme, never an unprompted +default. + +Flag `linear-gradient` or `radial-gradient` (or `bg-gradient-*` utilities) +as page or section backgrounds, and gradient-filled buttons. Also flag +`backdrop-filter: blur` / `backdrop-blur-*` used for glassmorphism cards. + +Flag assistant messages rendered as bubbles with initials-circle avatars +instead of the message components' plain-prose grammar. Flag emoji used as +icons or imagery in interface chrome. + +Flag stock template copy in headings: "Simple, transparent pricing", +"Welcome back", and interchangeable-with-a-competitor phrasing. Recommend +copy that states what this product specifically does. + +Do not flag pill-shaped controls or fluid clamp display headings — those +median patterns are deliberately shared here and their presence is fidelity. diff --git a/packages/vessel-react/fingerprint/checks/token-discipline.md b/packages/vessel-react/fingerprint/checks/token-discipline.md new file mode 100644 index 00000000..292df225 --- /dev/null +++ b/packages/vessel-react/fingerprint/checks/token-discipline.md @@ -0,0 +1,46 @@ +--- +name: Token discipline +description: Flags raw palette values, invented shadows, margin-driven rhythm, and role bypasses that break the token contract. +severity: high +references: + - contract.tokens + - contract.theming + - primitive.composition +--- + +Review changed component and style code by view, not just by file. + +Flag raw palette utilities (`bg-gray-*`, `text-gray-*`, `bg-white`, +`bg-black`) and literal hex, rgb, or hsl values in component code or inline +styles when a semantic role expresses the same thing. The authoring +vocabulary is `bg-background`, `text-muted-foreground`, `border-border`, and +the other role utilities. + +Flag the deprecated compatibility aliases (`background-*`, `text-*`, +`border-*` token families) in new code. They exist for migration, not +authoring. + +Flag chart hues (`chart-1` through `chart-5`) used outside data +visualization markup — on buttons, badges, headers, or backgrounds. Flag any +expressive color that is neither a gray, a status token, nor a chart role. + +Flag status colors used without their meaning: red for emphasis, green for +brand warmth, blue as an accent. Red is destructive/error, green success, +yellow warning, blue information — always. + +Flag `box-shadow` values (or arbitrary `shadow-[...]` utilities) that do not +use the elevation tiers (`--shadow-card`, `--shadow-popover`, +`--shadow-modal`) or a component-owned shadow (`--shadow-btn`, +`--shadow-mini`, `--shadow-kbd`). Do not accept a custom shadow because it +is visually close. Flag nested surfaces that stack elevation tiers. + +Flag new broad token aliases (`background-alt`, `border-strong`, +`surface-card`, and similar). Token extensions must be narrow and job-named. + +Flag margins on siblings inside a Stack, and arbitrary gap values +(`gap-[13px]`) that route around the gap scale. Rhythm comes from the +stack's gap prop; recommend changing the gap or splitting the stack. + +Flag buttons or text inputs restyled away from the control radius role, and +theme changes that rebind one control's radius without the others. Radius +coherence is contract, not preference. diff --git a/packages/vessel-react/fingerprint/contract.theming.md b/packages/vessel-react/fingerprint/contract.theming.md new file mode 100644 index 00000000..2c488756 --- /dev/null +++ b/packages/vessel-react/fingerprint/contract.theming.md @@ -0,0 +1,41 @@ +--- +description: "The theming seams — which token bindings a theme may rebind freely, and which relationships must hold in every theme." +materials: + - "**/styles/main.css" +--- + +Vessel ships vanilla: a deliberate default binding, not the language itself. +A theme rebinds values at the seams. The grammar beneath them is invariant. + +Free seams — rebind without ceremony: + +- The gray ramp, and the values behind every semantic role. +- The radius values: `--radius-pill`, `--radius-card`, `--radius-modal`, + `--radius-dropdown`, and friends. Pill-first is the vanilla default with + taste, not law; a theme may flatten controls. +- The shadow values inside each tier. +- The font stack. Vanilla stays on system fonts so no brand is baked in; + a theme brings its own typeface. +- The chart hues. + +Invariant grammar — a theme that breaks these is a fork, not a theme: + +- Radius coherence: controls share one radius role and contained surfaces + share another, and the system stays ordered. Rebinding buttons to a new + radius while inputs keep the old one is drift, not theming. +- Tier ordering: card reads below popover reads below modal in every theme. + Elevation must keep meaning interaction, whatever the shadow values are. +- Status semantics: red is destructive/error, green success, yellow warning, + blue information — in every theme. Status hues never become atmosphere. +- Role integrity: every foreground role stays legible on its paired surface + role. Rebind pairs together. +- Monochrome-first: whatever the ramp's hue temperature, the interface reads + as one quiet scale with meaning carried by structure. Expressive color + stays inside data visualization. +- Controls confirm with color and background shifts, not levitation, in + every theme. + +Rebind seams in the theme layer (`:root` / `.dark` overrides), never by +editing component classnames to route around a token. If a theme needs a hook +the seams don't expose, add a narrow job-named token — that is a contract +change and should be reviewed as one. diff --git a/packages/vessel-react/fingerprint/contract.tokens.md b/packages/vessel-react/fingerprint/contract.tokens.md new file mode 100644 index 00000000..1807b098 --- /dev/null +++ b/packages/vessel-react/fingerprint/contract.tokens.md @@ -0,0 +1,42 @@ +--- +description: The token pipeline — primitive values feed semantic roles, components author role-first, extensions stay narrow and job-named. +materials: + - "**/styles/main.css" + - "**/components/ui/**/*.tsx" +--- + +The token layer has a fixed shape that all component and styling work must +preserve: + +```text +primitive values (gray scale, utility colors) + -> semantic roles (shadcn names) + -> narrow job-named extensions + -> Tailwind utility bridge +``` + +Author against semantic roles first: `background`, `foreground`, `card`, +`popover`, `muted`, `accent`, `primary`, `secondary`, `destructive`, `border`, +`input`, `ring`, and the sidebar roles. In component code that means +`bg-background`, `text-muted-foreground`, `border-border` — never raw palette +utilities like `bg-gray-100`, and never literal hex, rgb, or hsl in component +code or inline styles. + +The base palette is monochrome gray. Red, green, yellow, and blue exist only +when meaning demands them: destructive/error, success, warning, information. +None of them are accents. The chart roles (`chart-1` through `chart-5`) are +the only sanctioned expressive hues, and they live inside data visualization — +they never leak into controls, badges, or chrome. + +Extensions to the token set must be narrow and job-named (composer surfaces, +message surfaces, tool/status affordances) — never broad duplicate aliases +such as `background-alt`, `text-alt`, `border-strong`, or `surface-card`. +Bridge a token into Tailwind only when component code should author it as a +utility class. + +Elevation is an interaction statement, not decoration. Surfaces pick a tier: +`--shadow-card` sits in flow, `--shadow-popover` floats above it, +`--shadow-modal` interrupts the task. Component shadows (`--shadow-btn`, +`--shadow-mini`, `--shadow-kbd`) belong to the components that own them. Never +write a custom `box-shadow` because a composition feels flat — fix the +structure instead. diff --git a/packages/vessel-react/fingerprint/glossary.md b/packages/vessel-react/fingerprint/glossary.md new file mode 100644 index 00000000..7205f5ad --- /dev/null +++ b/packages/vessel-react/fingerprint/glossary.md @@ -0,0 +1,29 @@ +--- +kinds: + - name: contract + - name: primitive + - name: pattern + - name: anti-goal + posture: guard +--- + +# contract + +An invariant of the vendored component set: the token pipeline, the theming +seams, and the relationships that must hold whatever values a theme binds. +Breaking a contract is a fork, not a customization. + +# primitive + +How a family of vendored components is meant to be used: its variant +vocabulary, its defaults, and the composition decisions the source encodes. + +# pattern + +Assembly grammar that spans components — the arrangements that make a screen +feel intentional instead of a pile of imported parts. + +# anti-goal + +What this component set never looks like: the measured defaults of unsteered +generation, rejected with replacements. diff --git a/packages/vessel-react/fingerprint/index.md b/packages/vessel-react/fingerprint/index.md new file mode 100644 index 00000000..35434fea --- /dev/null +++ b/packages/vessel-react/fingerprint/index.md @@ -0,0 +1,30 @@ +--- +description: "Always read first: what this vendored fingerprint is, what it governs, and that this repo's own nodes always win." +materials: + - "**/styles/main.css" +--- + +This fingerprint ships with the Vessel component set. It is not a brand. It is +the taste floor: the contracts, usage grammar, and anti-goals that keep the +vendored components coherent as this repo builds on and modifies them. + +The components are owned-after-copy, and so is this fingerprint. Edit these +nodes when this repo's decisions diverge; delete nodes that stop being true. +A node authored by this repo overrides anything here — these are reference +defaults, not law. + +The contract is narrow. Author against the semantic token roles, never raw +palette values. Compose from the vendored components before writing new ones. +When a new component or variant is needed, it should come from observed +repeated need, not anticipation — and it should be reviewed against the +contracts here. + +Vanilla is a binding, not the language. The theming contract names which +seams a theme may rebind and which relationships must hold regardless. Restyle +freely at the seams; do not fork the grammar silently. + +This fingerprint deliberately carries no product stance, flows, copy, or trust +obligations. Author those as this repo's own nodes, next to these. + +After vendoring, verify the materials globs in each node resolve against this +repo's actual component paths and tighten them, then run `ghost validate`. diff --git a/packages/vessel/.ghost/manifest.yml b/packages/vessel-react/fingerprint/manifest.yml similarity index 100% rename from packages/vessel/.ghost/manifest.yml rename to packages/vessel-react/fingerprint/manifest.yml diff --git a/packages/vessel-react/fingerprint/pattern.conversation.md b/packages/vessel-react/fingerprint/pattern.conversation.md new file mode 100644 index 00000000..fdfee086 --- /dev/null +++ b/packages/vessel-react/fingerprint/pattern.conversation.md @@ -0,0 +1,37 @@ +--- +description: Conversation grammar for AI surfaces — assistant prose on the page surface, quiet user containment, tools and reasoning as collapsed affordances. +materials: + - "**/components/ai-elements/conversation.tsx" + - "**/components/ai-elements/message.tsx" + - "**/components/ai-elements/prompt-input.tsx" + - "**/components/ai-elements/reasoning.tsx" + - "**/components/ai-elements/tool.tsx" + - "**/components/ai-elements/chain-of-thought.tsx" +--- + +A conversation is a document being written, not a chat skin. The grammar the +components encode: + +Assistant turns render as plain prose on the page surface — no bubble, no +border, no initials-circle avatar. The assistant is the document's author; +containing it in a balloon demotes it to a participant. + +User turns get quiet containment: a muted secondary container, right-aligned, +so the reader can scan whose words are whose. That asymmetry is the whole +message-identity system. Do not "balance" it by boxing both sides. + +Machinery is subordinate to prose. Reasoning, tool calls, chain-of-thought, +and sources render as collapsed, muted affordances the reader can open — never +as loud cards competing with the answer. Status is text and subtle motion +(shimmer), not colored badges per step. + +The composer is the one persistent control surface. It owns the bottom edge, +carries its own elevation, and holds the view's primary action. Nothing else +in the conversation column should compete with it for weight. + +Streaming is choreography, not spectacle: text appears in place, the scroll +follows the newest content until the user takes the wheel, and layout never +jumps as parts resolve. A stream that reflows the page is a broken stream. + +Empty conversation states say what this product does, in this product's +words — not "Start a conversation to see messages here" restyled. diff --git a/packages/vessel-react/fingerprint/primitive.composition.md b/packages/vessel-react/fingerprint/primitive.composition.md new file mode 100644 index 00000000..e0b4e77c --- /dev/null +++ b/packages/vessel-react/fingerprint/primitive.composition.md @@ -0,0 +1,41 @@ +--- +description: Stack, Surface, and Text — rhythm from the gap scale, elevation as interaction tiers, type from the variant vocabulary. +materials: + - "**/components/ui/stack.tsx" + - "**/components/ui/surface.tsx" + - "**/components/ui/text.tsx" + - "**/components/ui/card.tsx" + - "**/components/ui/separator.tsx" + - "**/components/ui/skeleton.tsx" +--- + +Three components carry most of a screen's composure: Stack owns rhythm, +Surface owns ground, Text owns hierarchy. Reach for them before writing +layout classnames by hand. + +Rhythm comes from Stack's gap scale, never from margins on siblings. When +spacing feels wrong, change the stack's gap or split into nested stacks with +different gaps — a margin patched onto a child is a rhythm leak, and a +one-off `gap-[13px]` is a scale violation. + +Surface makes ground explicit: a semantic role, a padding step, a radius +role, and an elevation tier, chosen together. The elevation tier is the +interaction statement — `card` sits in flow, `popover` floats above it, +`modal` interrupts. Do not stack elevation on nested surfaces; one shadow +per compositional idea. A view that needs three shadows needs fewer +surfaces. + +Text's variant vocabulary is the type scale: `display`, `headline`, `title`, +`body`, `label`, `mono`, with tone carried by the `tone` prop. Hierarchy +comes from variant and tone, not from ad-hoc `text-[17px]` or bolding body +copy. Muted tone is the workhorse for secondary information; if everything +is muted, nothing is. + +Empty states, loading states, and skeletons are part of the composition, not +afterthoughts. A skeleton mirrors the layout it replaces — same stacks, same +gaps — so the load resolves without reflow. + +The default temperature is quiet: white ground, hairline borders, generous +gaps, one idea per surface. When a composition feels flat, the fix is +structure — grouping, gap contrast, tone contrast — never a decorative +border, tint, or shadow. diff --git a/packages/vessel-react/fingerprint/primitive.controls.md b/packages/vessel-react/fingerprint/primitive.controls.md new file mode 100644 index 00000000..b74fc29f --- /dev/null +++ b/packages/vessel-react/fingerprint/primitive.controls.md @@ -0,0 +1,40 @@ +--- +description: Buttons, inputs, and forms — one primary action per view, quiet fields, focus rings as guidance, errors as facts at the field. +materials: + - "**/components/ui/button.tsx" + - "**/components/ui/button-group.tsx" + - "**/components/ui/input.tsx" + - "**/components/ui/input-group.tsx" + - "**/components/ui/textarea.tsx" + - "**/components/ui/select.tsx" + - "**/components/ui/form.tsx" + - "**/components/ui/label.tsx" +--- + +Controls make decisions obvious without making the view loud. + +One primary action per view. The `default` button variant is for the action +the screen exists to complete; everything else steps down the vocabulary — +`secondary`, `outline`, `ghost`, `link`. If two buttons both look primary, +the hierarchy failed. Do not invent variants outside this vocabulary; if a +new one is genuinely needed, it comes from repeated observed need and gets +added to the shared component, not inlined as one-off classnames. + +`destructive` is red and rare. It names the destructive act directly. Red is +never urgency, emphasis, or heat. + +Hover confirms with color and background shifts at the component's built-in +transition — never with `translateY` lift or a growing shadow. The source +encodes this; do not add motion on top. + +Fields are quiet until active: hairline border, muted placeholder, clear +label, focus ring on engagement. The ring is guidance, not decoration — do +not restyle it away or amplify it. + +Errors state facts next to the field that caused them, through the form +component's invalid states (`aria-invalid`, field messages). Do not move +field errors into modals, toasts, or generic banners when the user needs to +fix one input. + +Compose control rows with the group components rather than ad-hoc flex +wrappers, so spacing and radius joining stay owned by the system. diff --git a/packages/vessel/package.json b/packages/vessel-react/package.json similarity index 98% rename from packages/vessel/package.json rename to packages/vessel-react/package.json index ad983e07..1c141b9b 100644 --- a/packages/vessel/package.json +++ b/packages/vessel-react/package.json @@ -1,5 +1,5 @@ { - "name": "@design-intelligence/vessel", + "name": "@design-intelligence/vessel-react", "version": "0.1.0", "private": true, "type": "module", diff --git a/packages/vessel/public/r/accordion.json b/packages/vessel-react/public/r/accordion.json similarity index 100% rename from packages/vessel/public/r/accordion.json rename to packages/vessel-react/public/r/accordion.json diff --git a/packages/vessel/public/r/agent.json b/packages/vessel-react/public/r/agent.json similarity index 100% rename from packages/vessel/public/r/agent.json rename to packages/vessel-react/public/r/agent.json diff --git a/packages/vessel/public/r/alert-dialog.json b/packages/vessel-react/public/r/alert-dialog.json similarity index 100% rename from packages/vessel/public/r/alert-dialog.json rename to packages/vessel-react/public/r/alert-dialog.json diff --git a/packages/vessel/public/r/alert.json b/packages/vessel-react/public/r/alert.json similarity index 100% rename from packages/vessel/public/r/alert.json rename to packages/vessel-react/public/r/alert.json diff --git a/packages/vessel/public/r/artifact.json b/packages/vessel-react/public/r/artifact.json similarity index 100% rename from packages/vessel/public/r/artifact.json rename to packages/vessel-react/public/r/artifact.json diff --git a/packages/vessel/public/r/aspect-ratio.json b/packages/vessel-react/public/r/aspect-ratio.json similarity index 100% rename from packages/vessel/public/r/aspect-ratio.json rename to packages/vessel-react/public/r/aspect-ratio.json diff --git a/packages/vessel/public/r/attachments.json b/packages/vessel-react/public/r/attachments.json similarity index 100% rename from packages/vessel/public/r/attachments.json rename to packages/vessel-react/public/r/attachments.json diff --git a/packages/vessel/public/r/audio-player.json b/packages/vessel-react/public/r/audio-player.json similarity index 100% rename from packages/vessel/public/r/audio-player.json rename to packages/vessel-react/public/r/audio-player.json diff --git a/packages/vessel/public/r/avatar.json b/packages/vessel-react/public/r/avatar.json similarity index 100% rename from packages/vessel/public/r/avatar.json rename to packages/vessel-react/public/r/avatar.json diff --git a/packages/vessel/public/r/badge.json b/packages/vessel-react/public/r/badge.json similarity index 100% rename from packages/vessel/public/r/badge.json rename to packages/vessel-react/public/r/badge.json diff --git a/packages/vessel/public/r/breadcrumb.json b/packages/vessel-react/public/r/breadcrumb.json similarity index 100% rename from packages/vessel/public/r/breadcrumb.json rename to packages/vessel-react/public/r/breadcrumb.json diff --git a/packages/vessel/public/r/button-group.json b/packages/vessel-react/public/r/button-group.json similarity index 100% rename from packages/vessel/public/r/button-group.json rename to packages/vessel-react/public/r/button-group.json diff --git a/packages/vessel/public/r/button.json b/packages/vessel-react/public/r/button.json similarity index 100% rename from packages/vessel/public/r/button.json rename to packages/vessel-react/public/r/button.json diff --git a/packages/vessel/public/r/calendar.json b/packages/vessel-react/public/r/calendar.json similarity index 100% rename from packages/vessel/public/r/calendar.json rename to packages/vessel-react/public/r/calendar.json diff --git a/packages/vessel/public/r/canvas.json b/packages/vessel-react/public/r/canvas.json similarity index 100% rename from packages/vessel/public/r/canvas.json rename to packages/vessel-react/public/r/canvas.json diff --git a/packages/vessel/public/r/card.json b/packages/vessel-react/public/r/card.json similarity index 100% rename from packages/vessel/public/r/card.json rename to packages/vessel-react/public/r/card.json diff --git a/packages/vessel/public/r/carousel.json b/packages/vessel-react/public/r/carousel.json similarity index 100% rename from packages/vessel/public/r/carousel.json rename to packages/vessel-react/public/r/carousel.json diff --git a/packages/vessel/public/r/chain-of-thought.json b/packages/vessel-react/public/r/chain-of-thought.json similarity index 100% rename from packages/vessel/public/r/chain-of-thought.json rename to packages/vessel-react/public/r/chain-of-thought.json diff --git a/packages/vessel/public/r/chart.json b/packages/vessel-react/public/r/chart.json similarity index 100% rename from packages/vessel/public/r/chart.json rename to packages/vessel-react/public/r/chart.json diff --git a/packages/vessel/public/r/checkbox.json b/packages/vessel-react/public/r/checkbox.json similarity index 100% rename from packages/vessel/public/r/checkbox.json rename to packages/vessel-react/public/r/checkbox.json diff --git a/packages/vessel/public/r/checkpoint.json b/packages/vessel-react/public/r/checkpoint.json similarity index 100% rename from packages/vessel/public/r/checkpoint.json rename to packages/vessel-react/public/r/checkpoint.json diff --git a/packages/vessel/public/r/code-block.json b/packages/vessel-react/public/r/code-block.json similarity index 100% rename from packages/vessel/public/r/code-block.json rename to packages/vessel-react/public/r/code-block.json diff --git a/packages/vessel/public/r/collapsible.json b/packages/vessel-react/public/r/collapsible.json similarity index 100% rename from packages/vessel/public/r/collapsible.json rename to packages/vessel-react/public/r/collapsible.json diff --git a/packages/vessel/public/r/command.json b/packages/vessel-react/public/r/command.json similarity index 100% rename from packages/vessel/public/r/command.json rename to packages/vessel-react/public/r/command.json diff --git a/packages/vessel/public/r/commit.json b/packages/vessel-react/public/r/commit.json similarity index 100% rename from packages/vessel/public/r/commit.json rename to packages/vessel-react/public/r/commit.json diff --git a/packages/vessel/public/r/confirmation.json b/packages/vessel-react/public/r/confirmation.json similarity index 100% rename from packages/vessel/public/r/confirmation.json rename to packages/vessel-react/public/r/confirmation.json diff --git a/packages/vessel/public/r/connection.json b/packages/vessel-react/public/r/connection.json similarity index 100% rename from packages/vessel/public/r/connection.json rename to packages/vessel-react/public/r/connection.json diff --git a/packages/vessel/public/r/context-menu.json b/packages/vessel-react/public/r/context-menu.json similarity index 100% rename from packages/vessel/public/r/context-menu.json rename to packages/vessel-react/public/r/context-menu.json diff --git a/packages/vessel/public/r/context.json b/packages/vessel-react/public/r/context.json similarity index 100% rename from packages/vessel/public/r/context.json rename to packages/vessel-react/public/r/context.json diff --git a/packages/vessel/public/r/controls.json b/packages/vessel-react/public/r/controls.json similarity index 100% rename from packages/vessel/public/r/controls.json rename to packages/vessel-react/public/r/controls.json diff --git a/packages/vessel/public/r/conversation.json b/packages/vessel-react/public/r/conversation.json similarity index 100% rename from packages/vessel/public/r/conversation.json rename to packages/vessel-react/public/r/conversation.json diff --git a/packages/vessel/public/r/dialog.json b/packages/vessel-react/public/r/dialog.json similarity index 100% rename from packages/vessel/public/r/dialog.json rename to packages/vessel-react/public/r/dialog.json diff --git a/packages/vessel/public/r/drawer.json b/packages/vessel-react/public/r/drawer.json similarity index 100% rename from packages/vessel/public/r/drawer.json rename to packages/vessel-react/public/r/drawer.json diff --git a/packages/vessel/public/r/dropdown-menu.json b/packages/vessel-react/public/r/dropdown-menu.json similarity index 100% rename from packages/vessel/public/r/dropdown-menu.json rename to packages/vessel-react/public/r/dropdown-menu.json diff --git a/packages/vessel/public/r/edge.json b/packages/vessel-react/public/r/edge.json similarity index 100% rename from packages/vessel/public/r/edge.json rename to packages/vessel-react/public/r/edge.json diff --git a/packages/vessel/public/r/environment-variables.json b/packages/vessel-react/public/r/environment-variables.json similarity index 100% rename from packages/vessel/public/r/environment-variables.json rename to packages/vessel-react/public/r/environment-variables.json diff --git a/packages/vessel/public/r/file-tree.json b/packages/vessel-react/public/r/file-tree.json similarity index 100% rename from packages/vessel/public/r/file-tree.json rename to packages/vessel-react/public/r/file-tree.json diff --git a/packages/vessel/public/r/font-faces.json b/packages/vessel-react/public/r/font-faces.json similarity index 100% rename from packages/vessel/public/r/font-faces.json rename to packages/vessel-react/public/r/font-faces.json diff --git a/packages/vessel/public/r/form.json b/packages/vessel-react/public/r/form.json similarity index 100% rename from packages/vessel/public/r/form.json rename to packages/vessel-react/public/r/form.json diff --git a/packages/vessel/public/r/hover-card.json b/packages/vessel-react/public/r/hover-card.json similarity index 100% rename from packages/vessel/public/r/hover-card.json rename to packages/vessel-react/public/r/hover-card.json diff --git a/packages/vessel/public/r/image.json b/packages/vessel-react/public/r/image.json similarity index 100% rename from packages/vessel/public/r/image.json rename to packages/vessel-react/public/r/image.json diff --git a/packages/vessel/public/r/inline-citation.json b/packages/vessel-react/public/r/inline-citation.json similarity index 100% rename from packages/vessel/public/r/inline-citation.json rename to packages/vessel-react/public/r/inline-citation.json diff --git a/packages/vessel/public/r/input-group.json b/packages/vessel-react/public/r/input-group.json similarity index 100% rename from packages/vessel/public/r/input-group.json rename to packages/vessel-react/public/r/input-group.json diff --git a/packages/vessel/public/r/input-otp.json b/packages/vessel-react/public/r/input-otp.json similarity index 100% rename from packages/vessel/public/r/input-otp.json rename to packages/vessel-react/public/r/input-otp.json diff --git a/packages/vessel/public/r/input.json b/packages/vessel-react/public/r/input.json similarity index 100% rename from packages/vessel/public/r/input.json rename to packages/vessel-react/public/r/input.json diff --git a/packages/vessel/public/r/jsx-preview.json b/packages/vessel-react/public/r/jsx-preview.json similarity index 100% rename from packages/vessel/public/r/jsx-preview.json rename to packages/vessel-react/public/r/jsx-preview.json diff --git a/packages/vessel/public/r/label.json b/packages/vessel-react/public/r/label.json similarity index 100% rename from packages/vessel/public/r/label.json rename to packages/vessel-react/public/r/label.json diff --git a/packages/vessel/public/r/menubar.json b/packages/vessel-react/public/r/menubar.json similarity index 100% rename from packages/vessel/public/r/menubar.json rename to packages/vessel-react/public/r/menubar.json diff --git a/packages/vessel/public/r/message.json b/packages/vessel-react/public/r/message.json similarity index 100% rename from packages/vessel/public/r/message.json rename to packages/vessel-react/public/r/message.json diff --git a/packages/vessel/public/r/mic-selector.json b/packages/vessel-react/public/r/mic-selector.json similarity index 100% rename from packages/vessel/public/r/mic-selector.json rename to packages/vessel-react/public/r/mic-selector.json diff --git a/packages/vessel/public/r/model-selector.json b/packages/vessel-react/public/r/model-selector.json similarity index 100% rename from packages/vessel/public/r/model-selector.json rename to packages/vessel-react/public/r/model-selector.json diff --git a/packages/vessel/public/r/navigation-menu.json b/packages/vessel-react/public/r/navigation-menu.json similarity index 100% rename from packages/vessel/public/r/navigation-menu.json rename to packages/vessel-react/public/r/navigation-menu.json diff --git a/packages/vessel/public/r/node.json b/packages/vessel-react/public/r/node.json similarity index 100% rename from packages/vessel/public/r/node.json rename to packages/vessel-react/public/r/node.json diff --git a/packages/vessel/public/r/open-in-chat.json b/packages/vessel-react/public/r/open-in-chat.json similarity index 100% rename from packages/vessel/public/r/open-in-chat.json rename to packages/vessel-react/public/r/open-in-chat.json diff --git a/packages/vessel/public/r/package-info.json b/packages/vessel-react/public/r/package-info.json similarity index 100% rename from packages/vessel/public/r/package-info.json rename to packages/vessel-react/public/r/package-info.json diff --git a/packages/vessel/public/r/pagination.json b/packages/vessel-react/public/r/pagination.json similarity index 100% rename from packages/vessel/public/r/pagination.json rename to packages/vessel-react/public/r/pagination.json diff --git a/packages/vessel/public/r/panel.json b/packages/vessel-react/public/r/panel.json similarity index 100% rename from packages/vessel/public/r/panel.json rename to packages/vessel-react/public/r/panel.json diff --git a/packages/vessel/public/r/persona.json b/packages/vessel-react/public/r/persona.json similarity index 100% rename from packages/vessel/public/r/persona.json rename to packages/vessel-react/public/r/persona.json diff --git a/packages/vessel/public/r/plan.json b/packages/vessel-react/public/r/plan.json similarity index 100% rename from packages/vessel/public/r/plan.json rename to packages/vessel-react/public/r/plan.json diff --git a/packages/vessel/public/r/popover.json b/packages/vessel-react/public/r/popover.json similarity index 100% rename from packages/vessel/public/r/popover.json rename to packages/vessel-react/public/r/popover.json diff --git a/packages/vessel/public/r/progress.json b/packages/vessel-react/public/r/progress.json similarity index 100% rename from packages/vessel/public/r/progress.json rename to packages/vessel-react/public/r/progress.json diff --git a/packages/vessel/public/r/prompt-input.json b/packages/vessel-react/public/r/prompt-input.json similarity index 100% rename from packages/vessel/public/r/prompt-input.json rename to packages/vessel-react/public/r/prompt-input.json diff --git a/packages/vessel/public/r/queue.json b/packages/vessel-react/public/r/queue.json similarity index 100% rename from packages/vessel/public/r/queue.json rename to packages/vessel-react/public/r/queue.json diff --git a/packages/vessel/public/r/radio-group.json b/packages/vessel-react/public/r/radio-group.json similarity index 100% rename from packages/vessel/public/r/radio-group.json rename to packages/vessel-react/public/r/radio-group.json diff --git a/packages/vessel/public/r/reasoning.json b/packages/vessel-react/public/r/reasoning.json similarity index 100% rename from packages/vessel/public/r/reasoning.json rename to packages/vessel-react/public/r/reasoning.json diff --git a/packages/vessel/public/r/registry.json b/packages/vessel-react/public/r/registry.json similarity index 100% rename from packages/vessel/public/r/registry.json rename to packages/vessel-react/public/r/registry.json diff --git a/packages/vessel/public/r/resizable.json b/packages/vessel-react/public/r/resizable.json similarity index 100% rename from packages/vessel/public/r/resizable.json rename to packages/vessel-react/public/r/resizable.json diff --git a/packages/vessel/public/r/sandbox.json b/packages/vessel-react/public/r/sandbox.json similarity index 100% rename from packages/vessel/public/r/sandbox.json rename to packages/vessel-react/public/r/sandbox.json diff --git a/packages/vessel/public/r/schema-display.json b/packages/vessel-react/public/r/schema-display.json similarity index 100% rename from packages/vessel/public/r/schema-display.json rename to packages/vessel-react/public/r/schema-display.json diff --git a/packages/vessel/public/r/scroll-area.json b/packages/vessel-react/public/r/scroll-area.json similarity index 100% rename from packages/vessel/public/r/scroll-area.json rename to packages/vessel-react/public/r/scroll-area.json diff --git a/packages/vessel/public/r/select.json b/packages/vessel-react/public/r/select.json similarity index 100% rename from packages/vessel/public/r/select.json rename to packages/vessel-react/public/r/select.json diff --git a/packages/vessel/public/r/separator.json b/packages/vessel-react/public/r/separator.json similarity index 100% rename from packages/vessel/public/r/separator.json rename to packages/vessel-react/public/r/separator.json diff --git a/packages/vessel/public/r/sheet.json b/packages/vessel-react/public/r/sheet.json similarity index 100% rename from packages/vessel/public/r/sheet.json rename to packages/vessel-react/public/r/sheet.json diff --git a/packages/vessel/public/r/shimmer.json b/packages/vessel-react/public/r/shimmer.json similarity index 100% rename from packages/vessel/public/r/shimmer.json rename to packages/vessel-react/public/r/shimmer.json diff --git a/packages/vessel/public/r/sidebar.json b/packages/vessel-react/public/r/sidebar.json similarity index 100% rename from packages/vessel/public/r/sidebar.json rename to packages/vessel-react/public/r/sidebar.json diff --git a/packages/vessel/public/r/skeleton.json b/packages/vessel-react/public/r/skeleton.json similarity index 100% rename from packages/vessel/public/r/skeleton.json rename to packages/vessel-react/public/r/skeleton.json diff --git a/packages/vessel/public/r/slider.json b/packages/vessel-react/public/r/slider.json similarity index 100% rename from packages/vessel/public/r/slider.json rename to packages/vessel-react/public/r/slider.json diff --git a/packages/vessel/public/r/snippet.json b/packages/vessel-react/public/r/snippet.json similarity index 100% rename from packages/vessel/public/r/snippet.json rename to packages/vessel-react/public/r/snippet.json diff --git a/packages/vessel/public/r/sonner.json b/packages/vessel-react/public/r/sonner.json similarity index 100% rename from packages/vessel/public/r/sonner.json rename to packages/vessel-react/public/r/sonner.json diff --git a/packages/vessel/public/r/sources.json b/packages/vessel-react/public/r/sources.json similarity index 100% rename from packages/vessel/public/r/sources.json rename to packages/vessel-react/public/r/sources.json diff --git a/packages/vessel/public/r/speech-input.json b/packages/vessel-react/public/r/speech-input.json similarity index 100% rename from packages/vessel/public/r/speech-input.json rename to packages/vessel-react/public/r/speech-input.json diff --git a/packages/vessel/public/r/spinner.json b/packages/vessel-react/public/r/spinner.json similarity index 100% rename from packages/vessel/public/r/spinner.json rename to packages/vessel-react/public/r/spinner.json diff --git a/packages/vessel/public/r/stack-trace.json b/packages/vessel-react/public/r/stack-trace.json similarity index 100% rename from packages/vessel/public/r/stack-trace.json rename to packages/vessel-react/public/r/stack-trace.json diff --git a/packages/vessel/public/r/stack.json b/packages/vessel-react/public/r/stack.json similarity index 100% rename from packages/vessel/public/r/stack.json rename to packages/vessel-react/public/r/stack.json diff --git a/packages/vessel/public/r/styles-main.json b/packages/vessel-react/public/r/styles-main.json similarity index 100% rename from packages/vessel/public/r/styles-main.json rename to packages/vessel-react/public/r/styles-main.json diff --git a/packages/vessel/public/r/suggestion.json b/packages/vessel-react/public/r/suggestion.json similarity index 100% rename from packages/vessel/public/r/suggestion.json rename to packages/vessel-react/public/r/suggestion.json diff --git a/packages/vessel/public/r/surface.json b/packages/vessel-react/public/r/surface.json similarity index 100% rename from packages/vessel/public/r/surface.json rename to packages/vessel-react/public/r/surface.json diff --git a/packages/vessel/public/r/switch.json b/packages/vessel-react/public/r/switch.json similarity index 100% rename from packages/vessel/public/r/switch.json rename to packages/vessel-react/public/r/switch.json diff --git a/packages/vessel/public/r/table.json b/packages/vessel-react/public/r/table.json similarity index 100% rename from packages/vessel/public/r/table.json rename to packages/vessel-react/public/r/table.json diff --git a/packages/vessel/public/r/tabs.json b/packages/vessel-react/public/r/tabs.json similarity index 100% rename from packages/vessel/public/r/tabs.json rename to packages/vessel-react/public/r/tabs.json diff --git a/packages/vessel/public/r/task.json b/packages/vessel-react/public/r/task.json similarity index 100% rename from packages/vessel/public/r/task.json rename to packages/vessel-react/public/r/task.json diff --git a/packages/vessel/public/r/terminal.json b/packages/vessel-react/public/r/terminal.json similarity index 100% rename from packages/vessel/public/r/terminal.json rename to packages/vessel-react/public/r/terminal.json diff --git a/packages/vessel/public/r/test-results.json b/packages/vessel-react/public/r/test-results.json similarity index 100% rename from packages/vessel/public/r/test-results.json rename to packages/vessel-react/public/r/test-results.json diff --git a/packages/vessel/public/r/text.json b/packages/vessel-react/public/r/text.json similarity index 100% rename from packages/vessel/public/r/text.json rename to packages/vessel-react/public/r/text.json diff --git a/packages/vessel/public/r/textarea.json b/packages/vessel-react/public/r/textarea.json similarity index 100% rename from packages/vessel/public/r/textarea.json rename to packages/vessel-react/public/r/textarea.json diff --git a/packages/vessel/public/r/theme-midnight-luxe.json b/packages/vessel-react/public/r/theme-midnight-luxe.json similarity index 100% rename from packages/vessel/public/r/theme-midnight-luxe.json rename to packages/vessel-react/public/r/theme-midnight-luxe.json diff --git a/packages/vessel/public/r/theme-neon-brutalist.json b/packages/vessel-react/public/r/theme-neon-brutalist.json similarity index 100% rename from packages/vessel/public/r/theme-neon-brutalist.json rename to packages/vessel-react/public/r/theme-neon-brutalist.json diff --git a/packages/vessel/public/r/theme-ocean.json b/packages/vessel-react/public/r/theme-ocean.json similarity index 100% rename from packages/vessel/public/r/theme-ocean.json rename to packages/vessel-react/public/r/theme-ocean.json diff --git a/packages/vessel/public/r/theme-soft-pastel.json b/packages/vessel-react/public/r/theme-soft-pastel.json similarity index 100% rename from packages/vessel/public/r/theme-soft-pastel.json rename to packages/vessel-react/public/r/theme-soft-pastel.json diff --git a/packages/vessel/public/r/theme-warm-sand.json b/packages/vessel-react/public/r/theme-warm-sand.json similarity index 100% rename from packages/vessel/public/r/theme-warm-sand.json rename to packages/vessel-react/public/r/theme-warm-sand.json diff --git a/packages/vessel/public/r/toggle-group.json b/packages/vessel-react/public/r/toggle-group.json similarity index 100% rename from packages/vessel/public/r/toggle-group.json rename to packages/vessel-react/public/r/toggle-group.json diff --git a/packages/vessel/public/r/toggle.json b/packages/vessel-react/public/r/toggle.json similarity index 100% rename from packages/vessel/public/r/toggle.json rename to packages/vessel-react/public/r/toggle.json diff --git a/packages/vessel/public/r/tool.json b/packages/vessel-react/public/r/tool.json similarity index 100% rename from packages/vessel/public/r/tool.json rename to packages/vessel-react/public/r/tool.json diff --git a/packages/vessel/public/r/toolbar.json b/packages/vessel-react/public/r/toolbar.json similarity index 100% rename from packages/vessel/public/r/toolbar.json rename to packages/vessel-react/public/r/toolbar.json diff --git a/packages/vessel/public/r/tooltip.json b/packages/vessel-react/public/r/tooltip.json similarity index 100% rename from packages/vessel/public/r/tooltip.json rename to packages/vessel-react/public/r/tooltip.json diff --git a/packages/vessel/public/r/transcription.json b/packages/vessel-react/public/r/transcription.json similarity index 100% rename from packages/vessel/public/r/transcription.json rename to packages/vessel-react/public/r/transcription.json diff --git a/packages/vessel/public/r/utils.json b/packages/vessel-react/public/r/utils.json similarity index 100% rename from packages/vessel/public/r/utils.json rename to packages/vessel-react/public/r/utils.json diff --git a/packages/vessel/public/r/vessel-base.json b/packages/vessel-react/public/r/vessel-base.json similarity index 100% rename from packages/vessel/public/r/vessel-base.json rename to packages/vessel-react/public/r/vessel-base.json diff --git a/packages/vessel/public/r/voice-selector.json b/packages/vessel-react/public/r/voice-selector.json similarity index 100% rename from packages/vessel/public/r/voice-selector.json rename to packages/vessel-react/public/r/voice-selector.json diff --git a/packages/vessel/public/r/web-preview.json b/packages/vessel-react/public/r/web-preview.json similarity index 100% rename from packages/vessel/public/r/web-preview.json rename to packages/vessel-react/public/r/web-preview.json diff --git a/packages/vessel/registry.json b/packages/vessel-react/registry.json similarity index 100% rename from packages/vessel/registry.json rename to packages/vessel-react/registry.json diff --git a/packages/vessel/scripts/audit-agent-safety.mjs b/packages/vessel-react/scripts/audit-agent-safety.mjs similarity index 100% rename from packages/vessel/scripts/audit-agent-safety.mjs rename to packages/vessel-react/scripts/audit-agent-safety.mjs diff --git a/packages/vessel/scripts/build-base-vars.mjs b/packages/vessel-react/scripts/build-base-vars.mjs similarity index 100% rename from packages/vessel/scripts/build-base-vars.mjs rename to packages/vessel-react/scripts/build-base-vars.mjs diff --git a/packages/vessel/scripts/build-presets.mjs b/packages/vessel-react/scripts/build-presets.mjs similarity index 100% rename from packages/vessel/scripts/build-presets.mjs rename to packages/vessel-react/scripts/build-presets.mjs diff --git a/packages/vessel/scripts/generate-skills.mjs b/packages/vessel-react/scripts/generate-skills.mjs similarity index 100% rename from packages/vessel/scripts/generate-skills.mjs rename to packages/vessel-react/scripts/generate-skills.mjs diff --git a/packages/vessel/scripts/resolve-dts-paths.mjs b/packages/vessel-react/scripts/resolve-dts-paths.mjs similarity index 100% rename from packages/vessel/scripts/resolve-dts-paths.mjs rename to packages/vessel-react/scripts/resolve-dts-paths.mjs diff --git a/packages/vessel/src/components/ai-elements/agent.tsx b/packages/vessel-react/src/components/ai-elements/agent.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/agent.tsx rename to packages/vessel-react/src/components/ai-elements/agent.tsx diff --git a/packages/vessel/src/components/ai-elements/artifact.tsx b/packages/vessel-react/src/components/ai-elements/artifact.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/artifact.tsx rename to packages/vessel-react/src/components/ai-elements/artifact.tsx diff --git a/packages/vessel/src/components/ai-elements/attachments.tsx b/packages/vessel-react/src/components/ai-elements/attachments.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/attachments.tsx rename to packages/vessel-react/src/components/ai-elements/attachments.tsx diff --git a/packages/vessel/src/components/ai-elements/audio-player.tsx b/packages/vessel-react/src/components/ai-elements/audio-player.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/audio-player.tsx rename to packages/vessel-react/src/components/ai-elements/audio-player.tsx diff --git a/packages/vessel/src/components/ai-elements/canvas.tsx b/packages/vessel-react/src/components/ai-elements/canvas.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/canvas.tsx rename to packages/vessel-react/src/components/ai-elements/canvas.tsx diff --git a/packages/vessel/src/components/ai-elements/chain-of-thought.tsx b/packages/vessel-react/src/components/ai-elements/chain-of-thought.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/chain-of-thought.tsx rename to packages/vessel-react/src/components/ai-elements/chain-of-thought.tsx diff --git a/packages/vessel/src/components/ai-elements/checkpoint.tsx b/packages/vessel-react/src/components/ai-elements/checkpoint.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/checkpoint.tsx rename to packages/vessel-react/src/components/ai-elements/checkpoint.tsx diff --git a/packages/vessel/src/components/ai-elements/code-block.tsx b/packages/vessel-react/src/components/ai-elements/code-block.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/code-block.tsx rename to packages/vessel-react/src/components/ai-elements/code-block.tsx diff --git a/packages/vessel/src/components/ai-elements/commit.tsx b/packages/vessel-react/src/components/ai-elements/commit.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/commit.tsx rename to packages/vessel-react/src/components/ai-elements/commit.tsx diff --git a/packages/vessel/src/components/ai-elements/confirmation.tsx b/packages/vessel-react/src/components/ai-elements/confirmation.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/confirmation.tsx rename to packages/vessel-react/src/components/ai-elements/confirmation.tsx diff --git a/packages/vessel/src/components/ai-elements/connection.tsx b/packages/vessel-react/src/components/ai-elements/connection.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/connection.tsx rename to packages/vessel-react/src/components/ai-elements/connection.tsx diff --git a/packages/vessel/src/components/ai-elements/context.tsx b/packages/vessel-react/src/components/ai-elements/context.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/context.tsx rename to packages/vessel-react/src/components/ai-elements/context.tsx diff --git a/packages/vessel/src/components/ai-elements/controls.tsx b/packages/vessel-react/src/components/ai-elements/controls.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/controls.tsx rename to packages/vessel-react/src/components/ai-elements/controls.tsx diff --git a/packages/vessel/src/components/ai-elements/conversation.tsx b/packages/vessel-react/src/components/ai-elements/conversation.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/conversation.tsx rename to packages/vessel-react/src/components/ai-elements/conversation.tsx diff --git a/packages/vessel/src/components/ai-elements/edge.tsx b/packages/vessel-react/src/components/ai-elements/edge.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/edge.tsx rename to packages/vessel-react/src/components/ai-elements/edge.tsx diff --git a/packages/vessel/src/components/ai-elements/environment-variables.tsx b/packages/vessel-react/src/components/ai-elements/environment-variables.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/environment-variables.tsx rename to packages/vessel-react/src/components/ai-elements/environment-variables.tsx diff --git a/packages/vessel/src/components/ai-elements/file-tree.tsx b/packages/vessel-react/src/components/ai-elements/file-tree.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/file-tree.tsx rename to packages/vessel-react/src/components/ai-elements/file-tree.tsx diff --git a/packages/vessel/src/components/ai-elements/image.tsx b/packages/vessel-react/src/components/ai-elements/image.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/image.tsx rename to packages/vessel-react/src/components/ai-elements/image.tsx diff --git a/packages/vessel/src/components/ai-elements/inline-citation.tsx b/packages/vessel-react/src/components/ai-elements/inline-citation.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/inline-citation.tsx rename to packages/vessel-react/src/components/ai-elements/inline-citation.tsx diff --git a/packages/vessel/src/components/ai-elements/jsx-preview.tsx b/packages/vessel-react/src/components/ai-elements/jsx-preview.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/jsx-preview.tsx rename to packages/vessel-react/src/components/ai-elements/jsx-preview.tsx diff --git a/packages/vessel/src/components/ai-elements/message.tsx b/packages/vessel-react/src/components/ai-elements/message.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/message.tsx rename to packages/vessel-react/src/components/ai-elements/message.tsx diff --git a/packages/vessel/src/components/ai-elements/mic-selector.tsx b/packages/vessel-react/src/components/ai-elements/mic-selector.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/mic-selector.tsx rename to packages/vessel-react/src/components/ai-elements/mic-selector.tsx diff --git a/packages/vessel/src/components/ai-elements/model-selector.tsx b/packages/vessel-react/src/components/ai-elements/model-selector.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/model-selector.tsx rename to packages/vessel-react/src/components/ai-elements/model-selector.tsx diff --git a/packages/vessel/src/components/ai-elements/node.tsx b/packages/vessel-react/src/components/ai-elements/node.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/node.tsx rename to packages/vessel-react/src/components/ai-elements/node.tsx diff --git a/packages/vessel/src/components/ai-elements/open-in-chat.tsx b/packages/vessel-react/src/components/ai-elements/open-in-chat.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/open-in-chat.tsx rename to packages/vessel-react/src/components/ai-elements/open-in-chat.tsx diff --git a/packages/vessel/src/components/ai-elements/package-info.tsx b/packages/vessel-react/src/components/ai-elements/package-info.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/package-info.tsx rename to packages/vessel-react/src/components/ai-elements/package-info.tsx diff --git a/packages/vessel/src/components/ai-elements/panel.tsx b/packages/vessel-react/src/components/ai-elements/panel.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/panel.tsx rename to packages/vessel-react/src/components/ai-elements/panel.tsx diff --git a/packages/vessel/src/components/ai-elements/persona.tsx b/packages/vessel-react/src/components/ai-elements/persona.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/persona.tsx rename to packages/vessel-react/src/components/ai-elements/persona.tsx diff --git a/packages/vessel/src/components/ai-elements/plan.tsx b/packages/vessel-react/src/components/ai-elements/plan.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/plan.tsx rename to packages/vessel-react/src/components/ai-elements/plan.tsx diff --git a/packages/vessel/src/components/ai-elements/prompt-input.tsx b/packages/vessel-react/src/components/ai-elements/prompt-input.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/prompt-input.tsx rename to packages/vessel-react/src/components/ai-elements/prompt-input.tsx diff --git a/packages/vessel/src/components/ai-elements/queue.tsx b/packages/vessel-react/src/components/ai-elements/queue.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/queue.tsx rename to packages/vessel-react/src/components/ai-elements/queue.tsx diff --git a/packages/vessel/src/components/ai-elements/reasoning.tsx b/packages/vessel-react/src/components/ai-elements/reasoning.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/reasoning.tsx rename to packages/vessel-react/src/components/ai-elements/reasoning.tsx diff --git a/packages/vessel/src/components/ai-elements/sandbox.tsx b/packages/vessel-react/src/components/ai-elements/sandbox.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/sandbox.tsx rename to packages/vessel-react/src/components/ai-elements/sandbox.tsx diff --git a/packages/vessel/src/components/ai-elements/schema-display.tsx b/packages/vessel-react/src/components/ai-elements/schema-display.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/schema-display.tsx rename to packages/vessel-react/src/components/ai-elements/schema-display.tsx diff --git a/packages/vessel/src/components/ai-elements/shimmer.tsx b/packages/vessel-react/src/components/ai-elements/shimmer.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/shimmer.tsx rename to packages/vessel-react/src/components/ai-elements/shimmer.tsx diff --git a/packages/vessel/src/components/ai-elements/snippet.tsx b/packages/vessel-react/src/components/ai-elements/snippet.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/snippet.tsx rename to packages/vessel-react/src/components/ai-elements/snippet.tsx diff --git a/packages/vessel/src/components/ai-elements/sources.tsx b/packages/vessel-react/src/components/ai-elements/sources.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/sources.tsx rename to packages/vessel-react/src/components/ai-elements/sources.tsx diff --git a/packages/vessel/src/components/ai-elements/speech-input.tsx b/packages/vessel-react/src/components/ai-elements/speech-input.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/speech-input.tsx rename to packages/vessel-react/src/components/ai-elements/speech-input.tsx diff --git a/packages/vessel/src/components/ai-elements/stack-trace.tsx b/packages/vessel-react/src/components/ai-elements/stack-trace.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/stack-trace.tsx rename to packages/vessel-react/src/components/ai-elements/stack-trace.tsx diff --git a/packages/vessel/src/components/ai-elements/suggestion.tsx b/packages/vessel-react/src/components/ai-elements/suggestion.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/suggestion.tsx rename to packages/vessel-react/src/components/ai-elements/suggestion.tsx diff --git a/packages/vessel/src/components/ai-elements/task.tsx b/packages/vessel-react/src/components/ai-elements/task.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/task.tsx rename to packages/vessel-react/src/components/ai-elements/task.tsx diff --git a/packages/vessel/src/components/ai-elements/terminal.tsx b/packages/vessel-react/src/components/ai-elements/terminal.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/terminal.tsx rename to packages/vessel-react/src/components/ai-elements/terminal.tsx diff --git a/packages/vessel/src/components/ai-elements/test-results.tsx b/packages/vessel-react/src/components/ai-elements/test-results.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/test-results.tsx rename to packages/vessel-react/src/components/ai-elements/test-results.tsx diff --git a/packages/vessel/src/components/ai-elements/tool.tsx b/packages/vessel-react/src/components/ai-elements/tool.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/tool.tsx rename to packages/vessel-react/src/components/ai-elements/tool.tsx diff --git a/packages/vessel/src/components/ai-elements/toolbar.tsx b/packages/vessel-react/src/components/ai-elements/toolbar.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/toolbar.tsx rename to packages/vessel-react/src/components/ai-elements/toolbar.tsx diff --git a/packages/vessel/src/components/ai-elements/transcription.tsx b/packages/vessel-react/src/components/ai-elements/transcription.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/transcription.tsx rename to packages/vessel-react/src/components/ai-elements/transcription.tsx diff --git a/packages/vessel/src/components/ai-elements/voice-selector.tsx b/packages/vessel-react/src/components/ai-elements/voice-selector.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/voice-selector.tsx rename to packages/vessel-react/src/components/ai-elements/voice-selector.tsx diff --git a/packages/vessel/src/components/ai-elements/web-preview.tsx b/packages/vessel-react/src/components/ai-elements/web-preview.tsx similarity index 100% rename from packages/vessel/src/components/ai-elements/web-preview.tsx rename to packages/vessel-react/src/components/ai-elements/web-preview.tsx diff --git a/packages/vessel/src/components/theme/ThemeProvider.tsx b/packages/vessel-react/src/components/theme/ThemeProvider.tsx similarity index 100% rename from packages/vessel/src/components/theme/ThemeProvider.tsx rename to packages/vessel-react/src/components/theme/ThemeProvider.tsx diff --git a/packages/vessel/src/components/theme/ThemeToggle.tsx b/packages/vessel-react/src/components/theme/ThemeToggle.tsx similarity index 100% rename from packages/vessel/src/components/theme/ThemeToggle.tsx rename to packages/vessel-react/src/components/theme/ThemeToggle.tsx diff --git a/packages/vessel/src/components/ui/accordion.tsx b/packages/vessel-react/src/components/ui/accordion.tsx similarity index 100% rename from packages/vessel/src/components/ui/accordion.tsx rename to packages/vessel-react/src/components/ui/accordion.tsx diff --git a/packages/vessel/src/components/ui/alert-dialog.tsx b/packages/vessel-react/src/components/ui/alert-dialog.tsx similarity index 100% rename from packages/vessel/src/components/ui/alert-dialog.tsx rename to packages/vessel-react/src/components/ui/alert-dialog.tsx diff --git a/packages/vessel/src/components/ui/alert.tsx b/packages/vessel-react/src/components/ui/alert.tsx similarity index 100% rename from packages/vessel/src/components/ui/alert.tsx rename to packages/vessel-react/src/components/ui/alert.tsx diff --git a/packages/vessel/src/components/ui/aspect-ratio.tsx b/packages/vessel-react/src/components/ui/aspect-ratio.tsx similarity index 100% rename from packages/vessel/src/components/ui/aspect-ratio.tsx rename to packages/vessel-react/src/components/ui/aspect-ratio.tsx diff --git a/packages/vessel/src/components/ui/avatar.tsx b/packages/vessel-react/src/components/ui/avatar.tsx similarity index 100% rename from packages/vessel/src/components/ui/avatar.tsx rename to packages/vessel-react/src/components/ui/avatar.tsx diff --git a/packages/vessel/src/components/ui/badge.tsx b/packages/vessel-react/src/components/ui/badge.tsx similarity index 100% rename from packages/vessel/src/components/ui/badge.tsx rename to packages/vessel-react/src/components/ui/badge.tsx diff --git a/packages/vessel/src/components/ui/breadcrumb.tsx b/packages/vessel-react/src/components/ui/breadcrumb.tsx similarity index 100% rename from packages/vessel/src/components/ui/breadcrumb.tsx rename to packages/vessel-react/src/components/ui/breadcrumb.tsx diff --git a/packages/vessel/src/components/ui/button-group.tsx b/packages/vessel-react/src/components/ui/button-group.tsx similarity index 100% rename from packages/vessel/src/components/ui/button-group.tsx rename to packages/vessel-react/src/components/ui/button-group.tsx diff --git a/packages/vessel/src/components/ui/button.tsx b/packages/vessel-react/src/components/ui/button.tsx similarity index 100% rename from packages/vessel/src/components/ui/button.tsx rename to packages/vessel-react/src/components/ui/button.tsx diff --git a/packages/vessel/src/components/ui/calendar.tsx b/packages/vessel-react/src/components/ui/calendar.tsx similarity index 100% rename from packages/vessel/src/components/ui/calendar.tsx rename to packages/vessel-react/src/components/ui/calendar.tsx diff --git a/packages/vessel/src/components/ui/card.tsx b/packages/vessel-react/src/components/ui/card.tsx similarity index 100% rename from packages/vessel/src/components/ui/card.tsx rename to packages/vessel-react/src/components/ui/card.tsx diff --git a/packages/vessel/src/components/ui/carousel.tsx b/packages/vessel-react/src/components/ui/carousel.tsx similarity index 100% rename from packages/vessel/src/components/ui/carousel.tsx rename to packages/vessel-react/src/components/ui/carousel.tsx diff --git a/packages/vessel/src/components/ui/chart.tsx b/packages/vessel-react/src/components/ui/chart.tsx similarity index 100% rename from packages/vessel/src/components/ui/chart.tsx rename to packages/vessel-react/src/components/ui/chart.tsx diff --git a/packages/vessel/src/components/ui/checkbox.tsx b/packages/vessel-react/src/components/ui/checkbox.tsx similarity index 100% rename from packages/vessel/src/components/ui/checkbox.tsx rename to packages/vessel-react/src/components/ui/checkbox.tsx diff --git a/packages/vessel/src/components/ui/collapsible.tsx b/packages/vessel-react/src/components/ui/collapsible.tsx similarity index 100% rename from packages/vessel/src/components/ui/collapsible.tsx rename to packages/vessel-react/src/components/ui/collapsible.tsx diff --git a/packages/vessel/src/components/ui/command.tsx b/packages/vessel-react/src/components/ui/command.tsx similarity index 100% rename from packages/vessel/src/components/ui/command.tsx rename to packages/vessel-react/src/components/ui/command.tsx diff --git a/packages/vessel/src/components/ui/context-menu.tsx b/packages/vessel-react/src/components/ui/context-menu.tsx similarity index 100% rename from packages/vessel/src/components/ui/context-menu.tsx rename to packages/vessel-react/src/components/ui/context-menu.tsx diff --git a/packages/vessel/src/components/ui/dialog.tsx b/packages/vessel-react/src/components/ui/dialog.tsx similarity index 100% rename from packages/vessel/src/components/ui/dialog.tsx rename to packages/vessel-react/src/components/ui/dialog.tsx diff --git a/packages/vessel/src/components/ui/drawer.tsx b/packages/vessel-react/src/components/ui/drawer.tsx similarity index 100% rename from packages/vessel/src/components/ui/drawer.tsx rename to packages/vessel-react/src/components/ui/drawer.tsx diff --git a/packages/vessel/src/components/ui/dropdown-menu.tsx b/packages/vessel-react/src/components/ui/dropdown-menu.tsx similarity index 100% rename from packages/vessel/src/components/ui/dropdown-menu.tsx rename to packages/vessel-react/src/components/ui/dropdown-menu.tsx diff --git a/packages/vessel/src/components/ui/form.tsx b/packages/vessel-react/src/components/ui/form.tsx similarity index 100% rename from packages/vessel/src/components/ui/form.tsx rename to packages/vessel-react/src/components/ui/form.tsx diff --git a/packages/vessel/src/components/ui/hover-card.tsx b/packages/vessel-react/src/components/ui/hover-card.tsx similarity index 100% rename from packages/vessel/src/components/ui/hover-card.tsx rename to packages/vessel-react/src/components/ui/hover-card.tsx diff --git a/packages/vessel/src/components/ui/input-group.tsx b/packages/vessel-react/src/components/ui/input-group.tsx similarity index 100% rename from packages/vessel/src/components/ui/input-group.tsx rename to packages/vessel-react/src/components/ui/input-group.tsx diff --git a/packages/vessel/src/components/ui/input-otp.tsx b/packages/vessel-react/src/components/ui/input-otp.tsx similarity index 100% rename from packages/vessel/src/components/ui/input-otp.tsx rename to packages/vessel-react/src/components/ui/input-otp.tsx diff --git a/packages/vessel/src/components/ui/input.tsx b/packages/vessel-react/src/components/ui/input.tsx similarity index 100% rename from packages/vessel/src/components/ui/input.tsx rename to packages/vessel-react/src/components/ui/input.tsx diff --git a/packages/vessel/src/components/ui/label.tsx b/packages/vessel-react/src/components/ui/label.tsx similarity index 100% rename from packages/vessel/src/components/ui/label.tsx rename to packages/vessel-react/src/components/ui/label.tsx diff --git a/packages/vessel/src/components/ui/menubar.tsx b/packages/vessel-react/src/components/ui/menubar.tsx similarity index 100% rename from packages/vessel/src/components/ui/menubar.tsx rename to packages/vessel-react/src/components/ui/menubar.tsx diff --git a/packages/vessel/src/components/ui/navigation-menu.tsx b/packages/vessel-react/src/components/ui/navigation-menu.tsx similarity index 100% rename from packages/vessel/src/components/ui/navigation-menu.tsx rename to packages/vessel-react/src/components/ui/navigation-menu.tsx diff --git a/packages/vessel/src/components/ui/pagination.tsx b/packages/vessel-react/src/components/ui/pagination.tsx similarity index 100% rename from packages/vessel/src/components/ui/pagination.tsx rename to packages/vessel-react/src/components/ui/pagination.tsx diff --git a/packages/vessel/src/components/ui/popover.tsx b/packages/vessel-react/src/components/ui/popover.tsx similarity index 100% rename from packages/vessel/src/components/ui/popover.tsx rename to packages/vessel-react/src/components/ui/popover.tsx diff --git a/packages/vessel/src/components/ui/progress.tsx b/packages/vessel-react/src/components/ui/progress.tsx similarity index 100% rename from packages/vessel/src/components/ui/progress.tsx rename to packages/vessel-react/src/components/ui/progress.tsx diff --git a/packages/vessel/src/components/ui/radio-group.tsx b/packages/vessel-react/src/components/ui/radio-group.tsx similarity index 100% rename from packages/vessel/src/components/ui/radio-group.tsx rename to packages/vessel-react/src/components/ui/radio-group.tsx diff --git a/packages/vessel/src/components/ui/resizable.tsx b/packages/vessel-react/src/components/ui/resizable.tsx similarity index 100% rename from packages/vessel/src/components/ui/resizable.tsx rename to packages/vessel-react/src/components/ui/resizable.tsx diff --git a/packages/vessel/src/components/ui/scroll-area.tsx b/packages/vessel-react/src/components/ui/scroll-area.tsx similarity index 100% rename from packages/vessel/src/components/ui/scroll-area.tsx rename to packages/vessel-react/src/components/ui/scroll-area.tsx diff --git a/packages/vessel/src/components/ui/select.tsx b/packages/vessel-react/src/components/ui/select.tsx similarity index 100% rename from packages/vessel/src/components/ui/select.tsx rename to packages/vessel-react/src/components/ui/select.tsx diff --git a/packages/vessel/src/components/ui/separator.tsx b/packages/vessel-react/src/components/ui/separator.tsx similarity index 100% rename from packages/vessel/src/components/ui/separator.tsx rename to packages/vessel-react/src/components/ui/separator.tsx diff --git a/packages/vessel/src/components/ui/sheet.tsx b/packages/vessel-react/src/components/ui/sheet.tsx similarity index 100% rename from packages/vessel/src/components/ui/sheet.tsx rename to packages/vessel-react/src/components/ui/sheet.tsx diff --git a/packages/vessel/src/components/ui/sidebar.tsx b/packages/vessel-react/src/components/ui/sidebar.tsx similarity index 100% rename from packages/vessel/src/components/ui/sidebar.tsx rename to packages/vessel-react/src/components/ui/sidebar.tsx diff --git a/packages/vessel/src/components/ui/skeleton.tsx b/packages/vessel-react/src/components/ui/skeleton.tsx similarity index 100% rename from packages/vessel/src/components/ui/skeleton.tsx rename to packages/vessel-react/src/components/ui/skeleton.tsx diff --git a/packages/vessel/src/components/ui/slider.tsx b/packages/vessel-react/src/components/ui/slider.tsx similarity index 100% rename from packages/vessel/src/components/ui/slider.tsx rename to packages/vessel-react/src/components/ui/slider.tsx diff --git a/packages/vessel/src/components/ui/sonner.tsx b/packages/vessel-react/src/components/ui/sonner.tsx similarity index 100% rename from packages/vessel/src/components/ui/sonner.tsx rename to packages/vessel-react/src/components/ui/sonner.tsx diff --git a/packages/vessel/src/components/ui/spinner.tsx b/packages/vessel-react/src/components/ui/spinner.tsx similarity index 100% rename from packages/vessel/src/components/ui/spinner.tsx rename to packages/vessel-react/src/components/ui/spinner.tsx diff --git a/packages/vessel/src/components/ui/stack.tsx b/packages/vessel-react/src/components/ui/stack.tsx similarity index 100% rename from packages/vessel/src/components/ui/stack.tsx rename to packages/vessel-react/src/components/ui/stack.tsx diff --git a/packages/vessel/src/components/ui/surface.tsx b/packages/vessel-react/src/components/ui/surface.tsx similarity index 100% rename from packages/vessel/src/components/ui/surface.tsx rename to packages/vessel-react/src/components/ui/surface.tsx diff --git a/packages/vessel/src/components/ui/switch.tsx b/packages/vessel-react/src/components/ui/switch.tsx similarity index 100% rename from packages/vessel/src/components/ui/switch.tsx rename to packages/vessel-react/src/components/ui/switch.tsx diff --git a/packages/vessel/src/components/ui/table.tsx b/packages/vessel-react/src/components/ui/table.tsx similarity index 100% rename from packages/vessel/src/components/ui/table.tsx rename to packages/vessel-react/src/components/ui/table.tsx diff --git a/packages/vessel/src/components/ui/tabs.tsx b/packages/vessel-react/src/components/ui/tabs.tsx similarity index 100% rename from packages/vessel/src/components/ui/tabs.tsx rename to packages/vessel-react/src/components/ui/tabs.tsx diff --git a/packages/vessel/src/components/ui/text.tsx b/packages/vessel-react/src/components/ui/text.tsx similarity index 100% rename from packages/vessel/src/components/ui/text.tsx rename to packages/vessel-react/src/components/ui/text.tsx diff --git a/packages/vessel/src/components/ui/textarea.tsx b/packages/vessel-react/src/components/ui/textarea.tsx similarity index 100% rename from packages/vessel/src/components/ui/textarea.tsx rename to packages/vessel-react/src/components/ui/textarea.tsx diff --git a/packages/vessel/src/components/ui/toggle-group.tsx b/packages/vessel-react/src/components/ui/toggle-group.tsx similarity index 100% rename from packages/vessel/src/components/ui/toggle-group.tsx rename to packages/vessel-react/src/components/ui/toggle-group.tsx diff --git a/packages/vessel/src/components/ui/toggle.tsx b/packages/vessel-react/src/components/ui/toggle.tsx similarity index 100% rename from packages/vessel/src/components/ui/toggle.tsx rename to packages/vessel-react/src/components/ui/toggle.tsx diff --git a/packages/vessel/src/components/ui/tooltip.tsx b/packages/vessel-react/src/components/ui/tooltip.tsx similarity index 100% rename from packages/vessel/src/components/ui/tooltip.tsx rename to packages/vessel-react/src/components/ui/tooltip.tsx diff --git a/packages/vessel-react/src/fonts/HKGrotesk-Black.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-Black.woff2 new file mode 100644 index 00000000..3b84b31f Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-Black.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-Bold.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-Bold.woff2 new file mode 100644 index 00000000..b078c0b2 Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-Bold.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-ExtraBold.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-ExtraBold.woff2 new file mode 100644 index 00000000..c6161ef0 Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-ExtraBold.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-Light.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-Light.woff2 new file mode 100644 index 00000000..d9012bf0 Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-Light.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-Medium.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-Medium.woff2 new file mode 100644 index 00000000..252d7e88 Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-Medium.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-Regular.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-Regular.woff2 new file mode 100644 index 00000000..e90c63ff Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-Regular.woff2 differ diff --git a/packages/vessel-react/src/fonts/HKGrotesk-SemiBold.woff2 b/packages/vessel-react/src/fonts/HKGrotesk-SemiBold.woff2 new file mode 100644 index 00000000..e99f11d2 Binary files /dev/null and b/packages/vessel-react/src/fonts/HKGrotesk-SemiBold.woff2 differ diff --git a/packages/vessel/src/hooks/use-copy-to-clipboard.ts b/packages/vessel-react/src/hooks/use-copy-to-clipboard.ts similarity index 100% rename from packages/vessel/src/hooks/use-copy-to-clipboard.ts rename to packages/vessel-react/src/hooks/use-copy-to-clipboard.ts diff --git a/packages/vessel/src/hooks/use-intersection-observer.ts b/packages/vessel-react/src/hooks/use-intersection-observer.ts similarity index 100% rename from packages/vessel/src/hooks/use-intersection-observer.ts rename to packages/vessel-react/src/hooks/use-intersection-observer.ts diff --git a/packages/vessel/src/hooks/use-mobile.ts b/packages/vessel-react/src/hooks/use-mobile.ts similarity index 100% rename from packages/vessel/src/hooks/use-mobile.ts rename to packages/vessel-react/src/hooks/use-mobile.ts diff --git a/packages/vessel/src/hooks/use-scroll-reveal.ts b/packages/vessel-react/src/hooks/use-scroll-reveal.ts similarity index 100% rename from packages/vessel/src/hooks/use-scroll-reveal.ts rename to packages/vessel-react/src/hooks/use-scroll-reveal.ts diff --git a/packages/vessel/src/hooks/use-text-animator.tsx b/packages/vessel-react/src/hooks/use-text-animator.tsx similarity index 100% rename from packages/vessel/src/hooks/use-text-animator.tsx rename to packages/vessel-react/src/hooks/use-text-animator.tsx diff --git a/packages/vessel/src/index.ts b/packages/vessel-react/src/index.ts similarity index 100% rename from packages/vessel/src/index.ts rename to packages/vessel-react/src/index.ts diff --git a/packages/vessel/src/lib/theme-defaults.ts b/packages/vessel-react/src/lib/theme-defaults.ts similarity index 100% rename from packages/vessel/src/lib/theme-defaults.ts rename to packages/vessel-react/src/lib/theme-defaults.ts diff --git a/packages/vessel/src/lib/theme-presets.ts b/packages/vessel-react/src/lib/theme-presets.ts similarity index 100% rename from packages/vessel/src/lib/theme-presets.ts rename to packages/vessel-react/src/lib/theme-presets.ts diff --git a/packages/vessel/src/lib/theme-provider.tsx b/packages/vessel-react/src/lib/theme-provider.tsx similarity index 100% rename from packages/vessel/src/lib/theme-provider.tsx rename to packages/vessel-react/src/lib/theme-provider.tsx diff --git a/packages/vessel/src/lib/theme-utils.ts b/packages/vessel-react/src/lib/theme-utils.ts similarity index 100% rename from packages/vessel/src/lib/theme-utils.ts rename to packages/vessel-react/src/lib/theme-utils.ts diff --git a/packages/vessel/src/lib/utils.ts b/packages/vessel-react/src/lib/utils.ts similarity index 100% rename from packages/vessel/src/lib/utils.ts rename to packages/vessel-react/src/lib/utils.ts diff --git a/packages/vessel/src/mcp/bin.ts b/packages/vessel-react/src/mcp/bin.ts similarity index 100% rename from packages/vessel/src/mcp/bin.ts rename to packages/vessel-react/src/mcp/bin.ts diff --git a/packages/vessel/src/mcp/data.ts b/packages/vessel-react/src/mcp/data.ts similarity index 100% rename from packages/vessel/src/mcp/data.ts rename to packages/vessel-react/src/mcp/data.ts diff --git a/packages/vessel/src/mcp/resources.ts b/packages/vessel-react/src/mcp/resources.ts similarity index 100% rename from packages/vessel/src/mcp/resources.ts rename to packages/vessel-react/src/mcp/resources.ts diff --git a/packages/vessel/src/mcp/server.ts b/packages/vessel-react/src/mcp/server.ts similarity index 100% rename from packages/vessel/src/mcp/server.ts rename to packages/vessel-react/src/mcp/server.ts diff --git a/packages/vessel/src/mcp/tools.ts b/packages/vessel-react/src/mcp/tools.ts similarity index 100% rename from packages/vessel/src/mcp/tools.ts rename to packages/vessel-react/src/mcp/tools.ts diff --git a/packages/vessel/src/styles/font-faces.css b/packages/vessel-react/src/styles/font-faces.css similarity index 100% rename from packages/vessel/src/styles/font-faces.css rename to packages/vessel-react/src/styles/font-faces.css diff --git a/packages/vessel/src/styles/main.css b/packages/vessel-react/src/styles/main.css similarity index 100% rename from packages/vessel/src/styles/main.css rename to packages/vessel-react/src/styles/main.css diff --git a/packages/vessel/tsconfig.json b/packages/vessel-react/tsconfig.json similarity index 100% rename from packages/vessel/tsconfig.json rename to packages/vessel-react/tsconfig.json diff --git a/packages/vessel/tsconfig.lib.json b/packages/vessel-react/tsconfig.lib.json similarity index 100% rename from packages/vessel/tsconfig.lib.json rename to packages/vessel-react/tsconfig.lib.json diff --git a/packages/vessel/tsconfig.mcp.json b/packages/vessel-react/tsconfig.mcp.json similarity index 100% rename from packages/vessel/tsconfig.mcp.json rename to packages/vessel-react/tsconfig.mcp.json diff --git a/packages/vessel/vite.lib.config.ts b/packages/vessel-react/vite.lib.config.ts similarity index 100% rename from packages/vessel/vite.lib.config.ts rename to packages/vessel-react/vite.lib.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f0d39a2..f7ccfd40 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,7 +44,7 @@ importers: apps/docs: dependencies: - '@design-intelligence/vessel': + '@design-intelligence/vessel-react': specifier: workspace:* version: link:../../packages/vessel '@hookform/resolvers': @@ -316,7 +316,7 @@ importers: specifier: ^4.3.6 version: 4.3.6 - packages/vessel: + packages/vessel-react: dependencies: '@hookform/resolvers': specifier: ^5.2.2 diff --git a/scripts/check-packed-package.mjs b/scripts/check-packed-package.mjs index 0b28b049..c6745704 100644 --- a/scripts/check-packed-package.mjs +++ b/scripts/check-packed-package.mjs @@ -148,6 +148,39 @@ try { } run("pnpm", ["exec", "ghost", "lint", ".ghost"], { cwd: consumerDir }); + // The vessel-light body must survive packing: corpus, binary fonts, checks. + // Run from the consumer root (where the tarball is installed) and target a + // separate package dir — `pnpm exec` from a bare subdirectory would escape + // to any globally installed ghost. + const bodyPackageDir = join("body-consumer", ".ghost"); + const bodyInit = run( + "pnpm", + [ + "exec", + "ghost", + "init", + "--body", + "vessel-light", + "--package", + bodyPackageDir, + "--format", + "json", + ], + { cwd: consumerDir }, + ); + const bodyOutput = JSON.parse(bodyInit); + for (const required of [ + "manifest.yml", + "anti-goal.median.md", + "materials/fonts/HKGrotesk-Regular.woff2", + "checks/values.md", + ]) { + if (!bodyOutput.written?.includes(required)) { + fail(`packed ghost init --body vessel-light did not write ${required}`); + } + } + run("pnpm", ["exec", "ghost", "lint", bodyPackageDir], { cwd: consumerDir }); + for (const specifier of PUBLIC_IMPORTS) { runNode(`await import(${JSON.stringify(specifier)});`, consumerDir); } diff --git a/scripts/check-release-tarball.mjs b/scripts/check-release-tarball.mjs index 9af27706..ed8aa742 100644 --- a/scripts/check-release-tarball.mjs +++ b/scripts/check-release-tarball.mjs @@ -7,6 +7,7 @@ import { readdirSync, readFileSync, rmSync, + statSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -65,6 +66,10 @@ try { "package.json", "dist/bin.js", "dist/cli.js", + "dist/init-payloads/median/anti-goal.median.md", + "dist/init-payloads/median/median-tells.md", + "dist/init-payloads/vessel-light/manifest.yml", + "dist/init-payloads/vessel-light/materials/fonts/HKGrotesk-Regular.woff2", "node_modules/cac", "node_modules/yaml", "node_modules/zod", @@ -76,6 +81,20 @@ try { } } + // Binary payloads must survive packing byte-for-byte. + const fontRelative = "materials/fonts/HKGrotesk-Regular.woff2"; + const packedFont = statSync( + join(packageDir, "dist", "init-payloads", "vessel-light", fontRelative), + ); + const sourceFont = statSync( + join(ROOT, "packages", "vessel-light", ".ghost", fontRelative), + ); + if (packedFont.size !== sourceFont.size) { + fail( + `packed font size ${packedFont.size} does not match source ${sourceFont.size}`, + ); + } + const packedPkg = JSON.parse( readFileSync(join(packageDir, "package.json"), "utf8"), ); @@ -111,6 +130,34 @@ try { ); } + // The vessel-light body installs from the standalone tarball too. + const bodyDir = join(tmpRoot, "body-consumer"); + mkdirSync(bodyDir, { recursive: true }); + const bodyInit = run( + "node", + [ + join(packageDir, "dist", "bin.js"), + "init", + "--body", + "vessel-light", + "--format", + "json", + ], + { cwd: bodyDir }, + ); + const bodyOutput = JSON.parse(bodyInit); + for (const required of [ + "anti-goal.median.md", + "materials/fonts/HKGrotesk-Regular.woff2", + "checks/values.md", + ]) { + if (!bodyOutput.written?.includes(required)) { + fail( + `release tarball ghost init --body vessel-light did not write ${required}`, + ); + } + } + const topLevelEntries = readdirSync(extractDir); if (topLevelEntries.length !== 1 || topLevelEntries[0] !== "package") { fail("release tarball must extract to a single package/ directory"); diff --git a/scripts/check-terminology.mjs b/scripts/check-terminology.mjs index bc26466a..a5f0bd14 100644 --- a/scripts/check-terminology.mjs +++ b/scripts/check-terminology.mjs @@ -13,7 +13,7 @@ const ROOTS = [ "packages/ghost/package.json", "packages/ghost/src", "packages/ghost/test", - "packages/vessel/README.md", + "packages/vessel-react/README.md", ".ghost", ".changeset", ]; diff --git a/scripts/sync-init-payloads.mjs b/scripts/sync-init-payloads.mjs new file mode 100644 index 00000000..ef9b7871 --- /dev/null +++ b/scripts/sync-init-payloads.mjs @@ -0,0 +1,39 @@ +#!/usr/bin/env node +import { existsSync } from "node:fs"; +import { cp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const distInitPayloadsDir = join( + ROOT, + "packages", + "ghost", + "dist", + "init-payloads", +); + +await rm(distInitPayloadsDir, { recursive: true, force: true }); + +const sourceInitPayloadsDir = join( + ROOT, + "packages", + "ghost", + "src", + "init-payloads", +); +if (existsSync(sourceInitPayloadsDir)) { + await cp(sourceInitPayloadsDir, distInitPayloadsDir, { recursive: true }); +} + +await cp( + join(ROOT, "packages", "vessel-light", ".ghost"), + join(distInitPayloadsDir, "vessel-light"), + { + recursive: true, + filter(source) { + const name = source.split(/[\\/]/).at(-1); + return name !== ".events" && name !== ".gitignore"; + }, + }, +); diff --git a/tsconfig.json b/tsconfig.json index 8116dd06..1fd4370a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,6 +14,6 @@ "files": [], "references": [ { "path": "packages/ghost" }, - { "path": "packages/vessel/tsconfig.mcp.json" } + { "path": "packages/vessel-react/tsconfig.mcp.json" } ] }