diff --git a/.changeset/embedded-consumer-contract.md b/.changeset/embedded-consumer-contract.md new file mode 100644 index 00000000..10f330e8 --- /dev/null +++ b/.changeset/embedded-consumer-contract.md @@ -0,0 +1,25 @@ +--- +"@design-intelligence/ghost": minor +--- + +Embedded-consumer contract: hoist gather/pull semantics from the CLI into the +library, so governed hosts embedding Ghost consume semantics instead of +mirroring command code. + +- `buildGatherMenu(catalog, { includeWild })` in ghost-core returns + `{ entries, wildAvailable, wildIncluded }` — the gather menu semantics + (wild-posture gate + discoverability count) as a pure function; the CLI now + consumes it. +- `orderPulledNodes` / `steeringBucket` (steering order: index → concrete → + steady/wild → guard, stable within bucket) exported from ghost-core; the + pull command now consumes them. +- Observability event types (`GatherObservabilityEvent`, + `PullObservabilityEvent`, `PullMiss`, `GhostObservabilityEvent`, + `NewGhostObservabilityEvent`) and the pure `stampGhostEvent` helper exported + from `./fingerprint`, so embedders can record tape-shaped events in their own + receipts without writing into a fingerprint package directory. +- `LoadedCheck` exported from `./fingerprint`. +- `#ghost-core` / `#scan` import types conditions point at shipped `dist/` + declarations instead of unshipped `src/`. +- New `docs/embedded-consumers.md`: the contract page naming which behaviors + are semantics every consumer must honor vs. CLI presentation. diff --git a/docs/embedded-consumers.md b/docs/embedded-consumers.md new file mode 100644 index 00000000..eea38849 --- /dev/null +++ b/docs/embedded-consumers.md @@ -0,0 +1,135 @@ +# Embedded Consumer Contract + +This page is for governed hosts that embed Ghost as a library instead of asking +a human to run commands or an agent to drive the CLI. A UI-generation server, a +review-orchestration service, or another controlled host can load a fingerprint +package, present the same selection surface, and record its own receipts without +shelling out to `ghost gather` or `ghost pull` at serve time. + +Ghost has three consumer shapes: + +- a human at a terminal, reading CLI output; +- a bring-your-own-agent workflow, where an agent drives the CLI and reads the + emitted packet; +- an embedded host, which imports `@design-intelligence/ghost/core` and + `@design-intelligence/ghost/fingerprint` and owns presentation, selection, + authorization, and telemetry. + +The third shape must preserve the same load-bearing semantics. The CLI is one +transport for those semantics, not their source of truth. + +## Semantics and presentation + +| Contract | Library home | Embedded requirement | +| --- | --- | --- | +| Wild-posture gate | `buildGatherMenu(catalog, { includeWild })` returns `{ entries, wildAvailable, wildIncluded }`. | Exclude wild nodes by default. Include them only for an explicit per-request opt-in. Treat `wildAvailable` as discoverability count: hidden wild nodes may advertise that open territory exists, but their ids stay hidden until opted in. | +| Steering order | `orderPulledNodes(nodes)` and `steeringBucket(node)`. | Preserve the pull order: `index` first, then concrete nodes, then steady and wild nodes, then guard nodes. Ordering is stable within each bucket. Do not sort by historical usage, string distance, or model preference. | +| Checks are review assertions | `loadFingerprintPackage` returns `checks: Map` from `@design-intelligence/ghost/fingerprint`; check parsing and slicing live in `@design-intelligence/ghost/core`. | `gather` and `pull` never emit checks. Do not put check bodies in generation context. Route checks back to truth nodes through each check's `references`; they are advisory review assertions, not source material for generation. | +| `index` front door | The catalog node whose `id` or `slug` is `index`; bucketed by `steeringBucket`. | Treat `index` as the fingerprint front door. If selected, read it before other pulled nodes, regardless of request order. | +| Skeleton extraction | `stripSkeletonSections(body)` and `extractSkeletonFences(body)`. | Strip Skeleton sections from the prose body handed to generation, then emit extracted fences last as structure seeds for the artifact. The structure starts the artifact; it does not replace the truth text. | + +Presentation belongs to each transport. Markdown and JSON formatting, warning +phrases, help text, and process exit codes are CLI presentation. Embedded hosts +may render menus, misses, materials, and skeletons differently as long as the +contracts above stay intact. + +## Observability + +The `.ghost/.events` tape is CLI transport. It is not canonical state, not part +of the fingerprint semantics, and not a file embedded consumers should write. +Vendored fingerprint packages are read-only content fixtures; do not mutate the +package directory to append events, cache selections, or store receipts. + +Library consumers should construct the same event shapes exported from +`@design-intelligence/ghost/fingerprint`: `GatherObservabilityEvent`, +`PullObservabilityEvent`, `GhostObservabilityEvent`, +`NewGhostObservabilityEvent`, `PullMiss`, and `stampGhostEvent`. Record stamped +events in the host's own telemetry or receipt system, alongside the request id, +actor, policy decision, and selected package version. + +Pulse data is local tuning signal only. It is for authors to see what agents +reached for and improve descriptions. It must never become ranking, memory, or +canonical fingerprint state. Do not feed historical hit rates back into +`buildGatherMenu`, `orderPulledNodes`, or any other menu ordering path. + +## Minimal consumption sketch + +```ts +import { + buildGatherMenu, + orderPulledNodes, + stripSkeletonSections, + extractSkeletonFences, +} from "@design-intelligence/ghost/core"; +import { + loadFingerprintPackage, + resolveFingerprintPackage, + stampGhostEvent, + type GhostObservabilityEvent, + type NewGhostObservabilityEvent, +} from "@design-intelligence/ghost/fingerprint"; + +async function buildGhostPacket(input: { + packageDir?: string; + request: { ask?: string; allowWild?: boolean }; + selectGhostNodes: (entries: ReturnType["entries"]) => + Promise; + recordGhostEvent: (event: GhostObservabilityEvent) => void; +}) { + const paths = resolveFingerprintPackage(input.packageDir, process.cwd()); + const loaded = await loadFingerprintPackage(paths); + + const menu = buildGatherMenu(loaded.catalog, { + includeWild: input.request.allowWild === true, + }); + const gatherEvent = { + event: "gather", + ...(input.request.ask ? { ask: input.request.ask } : {}), + menu: menu.entries.map((entry) => entry.id), + wild: input.request.allowWild === true, + wildIds: menu.entries.filter((entry) => entry.wild).map((entry) => entry.id), + } satisfies NewGhostObservabilityEvent; + input.recordGhostEvent(stampGhostEvent(gatherEvent)); + + // Host policy, UI, or a host agent selects ids from menu.entries. + const selectedIds = await input.selectGhostNodes(menu.entries); + const selectedNodes = selectedIds.flatMap((id) => { + const node = loaded.catalog.nodes.get(id); + return node ? [node] : []; + }); + + const pulled = orderPulledNodes(selectedNodes); + const generationContext = pulled.map((node) => ({ + id: node.id, + description: node.description, + body: stripSkeletonSections(node.body).trim(), + materials: node.materials, + })); + const skeletons = pulled.flatMap((node) => + extractSkeletonFences(node.body).map((fence) => ({ + nodeId: node.id, + info: fence.info, + content: fence.content, + })), + ); + + const wildIds = pulled.filter((node) => node.wild).map((node) => node.id); + const pullEvent = { + event: "pull", + ids: pulled.map((node) => node.id), + ...(wildIds.length > 0 ? { wildIds } : {}), + } satisfies NewGhostObservabilityEvent; + input.recordGhostEvent(stampGhostEvent(pullEvent)); + + return { generationContext, skeletons }; +} +``` + +## Embedders must not + +- shell out to the CLI at serve time; +- include wild nodes without an explicit per-request opt-in; +- expose hidden wild-node ids before that opt-in; +- surface check bodies to a generator; +- mutate a fingerprint package directory, including `.ghost/.events`; +- use pulse or historical hit rates to rank, remember, or rewrite the menu. diff --git a/packages/ghost/package.json b/packages/ghost/package.json index 97b5ed0e..defffd5b 100644 --- a/packages/ghost/package.json +++ b/packages/ghost/package.json @@ -31,11 +31,11 @@ }, "imports": { "#ghost-core": { - "types": "./src/ghost-core/index.ts", + "types": "./dist/ghost-core/index.d.ts", "import": "./dist/ghost-core/index.js" }, "#scan": { - "types": "./src/scan/index.ts", + "types": "./dist/scan/index.d.ts", "import": "./dist/scan/index.js" } }, diff --git a/packages/ghost/src/commands/gather-command.ts b/packages/ghost/src/commands/gather-command.ts index b22af377..76be1b03 100644 --- a/packages/ghost/src/commands/gather-command.ts +++ b/packages/ghost/src/commands/gather-command.ts @@ -1,7 +1,7 @@ import { readFile } from "node:fs/promises"; import type { CAC } from "cac"; import { - buildCatalogMenu, + buildGatherMenu, type CatalogMenuEntry, parseGlossary, } from "#ghost-core"; @@ -46,12 +46,9 @@ export function registerGatherCommand(cli: CAC): void { const ask = normalizeAsk(askParts); const paths = resolveFingerprintPackage(opts.package, process.cwd()); const loaded = await loadFingerprintPackage(paths); - const wildIds = [...loaded.catalog.nodes.values()] - .filter((node) => node.wild) - .map((node) => node.id) - .sort(); const includeWild = Boolean(opts.wild); - const menu = buildCatalogMenu(loaded.catalog, { includeWild }); + const gatherMenu = buildGatherMenu(loaded.catalog, { includeWild }); + const menu = gatherMenu.entries; const exposedWildIds = menu .filter((entry) => entry.wild) .map((entry) => entry.id); @@ -75,8 +72,8 @@ export function registerGatherCommand(cli: CAC): void { coverage: menuCoverage(menu), ...(kinds.length > 0 ? { kinds } : {}), nodes: menu, - ...(wildIds.length > 0 && !includeWild - ? { wildAvailable: wildIds.length } + ...(gatherMenu.wildAvailable > 0 + ? { wildAvailable: gatherMenu.wildAvailable } : {}), }, null, @@ -87,8 +84,8 @@ export function registerGatherCommand(cli: CAC): void { process.stdout.write( formatMenuMarkdown(menu, kinds, { ask, - wildAvailable: includeWild ? 0 : wildIds.length, - wildIncluded: includeWild && exposedWildIds.length > 0, + wildAvailable: gatherMenu.wildAvailable, + wildIncluded: gatherMenu.wildIncluded, }), ); } diff --git a/packages/ghost/src/commands/pull-command.ts b/packages/ghost/src/commands/pull-command.ts index c458b28c..d7536a5b 100644 --- a/packages/ghost/src/commands/pull-command.ts +++ b/packages/ghost/src/commands/pull-command.ts @@ -5,6 +5,7 @@ import { extractSkeletonFences, type GhostCatalogNode, type MaterialTransportResult, + orderPulledNodes, resolveLocalMaterialLocator, stripSkeletonSections, type TransportedMaterial, @@ -179,21 +180,6 @@ async function resolvePulledNodes( ); } -function orderPulledNodes(nodes: GhostCatalogNode[]): GhostCatalogNode[] { - return nodes - .map((node, index) => ({ node, index, bucket: steeringBucket(node) })) - .sort((a, b) => a.bucket - b.bucket || a.index - b.index) - .map((entry) => entry.node); -} - -function steeringBucket(node: GhostCatalogNode): number { - if (node.id === "index" || node.slug === "index") return 0; - if (node.guard) return 3; - if (node.wild) return 2; - if (node.concrete) return 1; - return 2; -} - function locatorOnlyMaterials( locators: string[] | undefined, repoRoot: string, diff --git a/packages/ghost/src/fingerprint.ts b/packages/ghost/src/fingerprint.ts index 5a7ed793..f60df368 100644 --- a/packages/ghost/src/fingerprint.ts +++ b/packages/ghost/src/fingerprint.ts @@ -1,3 +1,12 @@ +export type { + GatherObservabilityEvent, + GhostObservabilityEvent, + NewGhostObservabilityEvent, + PullMiss, + PullObservabilityEvent, +} from "./observability-events.js"; +export { stampGhostEvent } from "./observability-events.js"; +export type { LoadedCheck } from "./scan/check-files.js"; export { FINGERPRINT_MANIFEST_FILENAME, FINGERPRINT_PACKAGE_DIR, diff --git a/packages/ghost/src/ghost-core/catalog/index.ts b/packages/ghost/src/ghost-core/catalog/index.ts index ae97f029..825e2ebf 100644 --- a/packages/ghost/src/ghost-core/catalog/index.ts +++ b/packages/ghost/src/ghost-core/catalog/index.ts @@ -12,7 +12,11 @@ export { export { closestIds } from "./closest.js"; export { type BuildCatalogMenuOptions, + type BuildGatherMenuOptions, buildCatalogMenu, + buildGatherMenu, type CatalogMenuEntry, + type GatherMenu, } from "./menu.js"; +export { orderPulledNodes, steeringBucket } from "./steering.js"; export type { GhostCatalog, GhostCatalogNode } from "./types.js"; diff --git a/packages/ghost/src/ghost-core/catalog/menu.ts b/packages/ghost/src/ghost-core/catalog/menu.ts index 443d7030..1bd2d64d 100644 --- a/packages/ghost/src/ghost-core/catalog/menu.ts +++ b/packages/ghost/src/ghost-core/catalog/menu.ts @@ -1,5 +1,15 @@ import type { GhostCatalog } from "./types.js"; +export interface BuildGatherMenuOptions { + includeWild?: boolean; +} + +export interface GatherMenu { + entries: CatalogMenuEntry[]; + wildAvailable: number; + wildIncluded: boolean; +} + export interface BuildCatalogMenuOptions { /** Include kinds that declare `posture: wild`; default menus omit them. */ includeWild?: boolean; @@ -60,3 +70,20 @@ export function buildCatalogMenu( return entries.sort((a, b) => a.id.localeCompare(b.id)); } + +export function buildGatherMenu( + catalog: GhostCatalog, + options: BuildGatherMenuOptions = {}, +): GatherMenu { + const includeWild = Boolean(options.includeWild); + const entries = buildCatalogMenu(catalog, { includeWild }); + const wildCount = [...catalog.nodes.values()].filter( + (node) => node.wild, + ).length; + + return { + entries, + wildAvailable: includeWild ? 0 : wildCount, + wildIncluded: includeWild && entries.some((entry) => entry.wild), + }; +} diff --git a/packages/ghost/src/ghost-core/catalog/steering.ts b/packages/ghost/src/ghost-core/catalog/steering.ts new file mode 100644 index 00000000..4c16ada5 --- /dev/null +++ b/packages/ghost/src/ghost-core/catalog/steering.ts @@ -0,0 +1,18 @@ +import type { GhostCatalogNode } from "./types.js"; + +export function orderPulledNodes( + nodes: GhostCatalogNode[], +): GhostCatalogNode[] { + return nodes + .map((node, index) => ({ node, index, bucket: steeringBucket(node) })) + .sort((a, b) => a.bucket - b.bucket || a.index - b.index) + .map((entry) => entry.node); +} + +export function steeringBucket(node: GhostCatalogNode): number { + if (node.id === "index" || node.slug === "index") return 0; + if (node.guard) return 3; + if (node.wild) return 2; + if (node.concrete) return 1; + return 2; +} diff --git a/packages/ghost/src/ghost-core/index.ts b/packages/ghost/src/ghost-core/index.ts index cb06e4e2..9b664884 100644 --- a/packages/ghost/src/ghost-core/index.ts +++ b/packages/ghost/src/ghost-core/index.ts @@ -5,12 +5,17 @@ export { type AssembleCatalogInput, assembleCatalog, type BuildCatalogMenuOptions, + type BuildGatherMenuOptions, buildCatalogMenu, + buildGatherMenu, type CatalogMenuEntry, closestIds, + type GatherMenu, type GhostCatalog, type GhostCatalogNode, + orderPulledNodes, type PlacedNode, + steeringBucket, } from "./catalog/index.js"; // --- Check (ghost.check/v1) — markdown checks, agent-evaluated --- export { diff --git a/packages/ghost/src/observability-events.ts b/packages/ghost/src/observability-events.ts index d2b492ee..c57aaa98 100644 --- a/packages/ghost/src/observability-events.ts +++ b/packages/ghost/src/observability-events.ts @@ -4,6 +4,12 @@ import { GHOST_EVENTS_FILENAME } from "./scan/constants.js"; const FIRST_WRITE_NOTICE = `ghost: logging selection events locally to .ghost/${GHOST_EVENTS_FILENAME} (gitignored; never leaves your machine). Summarize with \`ghost pulse\`.`; +/** + * Gather event shape used by Ghost observability. The CLI appends these events + * to `.ghost/.events`; library consumers may construct the same shapes for + * their own telemetry, but must never write into a fingerprint package + * directory. + */ export type GatherObservabilityEvent = { ts: string; event: "gather"; @@ -14,11 +20,23 @@ export type GatherObservabilityEvent = { wildIds?: string[]; }; +/** + * Pull miss shape used by Ghost observability. The CLI appends enclosing pull + * events to `.ghost/.events`; library consumers may construct the same shapes + * for their own telemetry, but must never write into a fingerprint package + * directory. + */ export type PullMiss = { requested: string; suggested: string[]; }; +/** + * Pull event shape used by Ghost observability. The CLI appends these events + * to `.ghost/.events`; library consumers may construct the same shapes for + * their own telemetry, but must never write into a fingerprint package + * directory. + */ export type PullObservabilityEvent = { ts: string; event: "pull"; @@ -29,19 +47,36 @@ export type PullObservabilityEvent = { omittedMaterials?: number; }; +/** + * Timestamped Ghost observability event. The CLI appends these events to + * `.ghost/.events`; library consumers may construct the same shapes for their + * own telemetry, but must never write into a fingerprint package directory. + */ export type GhostObservabilityEvent = | GatherObservabilityEvent | PullObservabilityEvent; +/** + * Untimestamped Ghost observability event input. The CLI timestamps and appends + * these events to `.ghost/.events`; library consumers may construct the same + * shapes for their own telemetry, but must never write into a fingerprint + * package directory. + */ export type NewGhostObservabilityEvent = | Omit | Omit; +export function stampGhostEvent( + event: NewGhostObservabilityEvent, +): GhostObservabilityEvent { + return { ts: new Date().toISOString(), ...event }; +} + export async function appendGhostEvent( packageDir: string, event: NewGhostObservabilityEvent, ): Promise { - const line = `${JSON.stringify({ ts: new Date().toISOString(), ...event })}\n`; + const line = `${JSON.stringify(stampGhostEvent(event))}\n`; const tapePath = join(packageDir, GHOST_EVENTS_FILENAME); try { const isFirstWrite = await access(tapePath).then( diff --git a/packages/ghost/test/ghost-core/catalog-menu.test.ts b/packages/ghost/test/ghost-core/catalog-menu.test.ts new file mode 100644 index 00000000..9e192623 --- /dev/null +++ b/packages/ghost/test/ghost-core/catalog-menu.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { + buildGatherMenu, + type GhostCatalog, + type GhostCatalogNode, + orderPulledNodes, + steeringBucket, +} from "../../src/ghost-core/index.js"; + +function node( + id: string, + extra: Partial = {}, +): GhostCatalogNode { + return { + id, + slug: id.split(".").at(-1) ?? id, + concrete: false, + hasSkeleton: false, + posture: "steady", + body: "Prose.", + ...extra, + }; +} + +function catalog(nodes: GhostCatalogNode[]): GhostCatalog { + return { nodes: new Map(nodes.map((entry) => [entry.id, entry])) }; +} + +describe("buildGatherMenu", () => { + it("hides wild nodes by default and counts them as available", () => { + const menu = buildGatherMenu( + catalog([ + node("principle.trust"), + node("provocation.noise", { posture: "wild", wild: true }), + ]), + ); + + expect(menu.entries.map((entry) => entry.id)).toEqual(["principle.trust"]); + expect(menu.wildAvailable).toBe(1); + expect(menu.wildIncluded).toBe(false); + }); + + it("includes wild nodes on request and marks wildIncluded true", () => { + const menu = buildGatherMenu( + catalog([ + node("principle.trust"), + node("provocation.noise", { posture: "wild", wild: true }), + ]), + { includeWild: true }, + ); + + expect(menu.entries.map((entry) => entry.id)).toEqual([ + "principle.trust", + "provocation.noise", + ]); + expect( + menu.entries.find((entry) => entry.id === "provocation.noise"), + ).toMatchObject({ wild: true }); + expect(menu.wildAvailable).toBe(0); + expect(menu.wildIncluded).toBe(true); + }); + + it("reports zero wildAvailable when no wild nodes exist", () => { + const menu = buildGatherMenu( + catalog([node("index", { slug: "index" }), node("principle.trust")]), + ); + + expect(menu.wildAvailable).toBe(0); + expect(menu.wildIncluded).toBe(false); + }); +}); + +describe("steering order", () => { + it("assigns buckets for index, concrete, wild/default, and guard nodes", () => { + expect(steeringBucket(node("index", { slug: "elsewhere" }))).toBe(0); + expect(steeringBucket(node("principle.index", { slug: "index" }))).toBe(0); + expect(steeringBucket(node("asset.tokens", { concrete: true }))).toBe(1); + expect(steeringBucket(node("provocation.noise", { wild: true }))).toBe(2); + expect(steeringBucket(node("principle.trust"))).toBe(2); + expect(steeringBucket(node("anti-goal.generic", { guard: true }))).toBe(3); + }); + + it("orders by steering bucket while preserving original order within each bucket", () => { + const guard = node("anti-goal.generic", { guard: true }); + const defaultNode = node("principle.trust"); + const concreteA = node("asset.tokens", { concrete: true }); + const index = node("index", { slug: "index" }); + const wild = node("provocation.noise", { posture: "wild", wild: true }); + const concreteB = node("pattern.card", { concrete: true }); + + expect( + orderPulledNodes([ + guard, + defaultNode, + concreteA, + index, + wild, + concreteB, + ]).map((entry) => entry.id), + ).toEqual([ + "index", + "asset.tokens", + "pattern.card", + "principle.trust", + "provocation.noise", + "anti-goal.generic", + ]); + }); +}); diff --git a/packages/ghost/test/observability-events.test.ts b/packages/ghost/test/observability-events.test.ts new file mode 100644 index 00000000..59916ac4 --- /dev/null +++ b/packages/ghost/test/observability-events.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { stampGhostEvent } from "../src/observability-events.js"; + +describe("stampGhostEvent", () => { + it("adds an ISO timestamp and preserves the event shape", () => { + const before = Date.now(); + + const stamped = stampGhostEvent({ + event: "gather", + ask: "checkout", + menu: ["checkout/principle.trust"], + materials: ["checkout/copy.md"], + wild: true, + wildIds: ["checkout/provocation.speed"], + }); + + const after = Date.now(); + const stampedTime = Date.parse(stamped.ts); + + expect(stamped).toEqual({ + ts: expect.any(String), + event: "gather", + ask: "checkout", + menu: ["checkout/principle.trust"], + materials: ["checkout/copy.md"], + wild: true, + wildIds: ["checkout/provocation.speed"], + }); + expect(stamped.ts).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + expect(stampedTime).toBeGreaterThanOrEqual(before); + expect(stampedTime).toBeLessThanOrEqual(after); + }); +});