From deac499808c7a93f906f3bbd70d9037358899ba8 Mon Sep 17 00:00:00 2001 From: John Siracusa Date: Wed, 29 Jul 2026 23:32:46 -0400 Subject: [PATCH 1/4] refactor(core): extract retrieval into search.mjs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ranking behind `search` and `find_captures` lived inside mcp-server.mjs, which parses argv and can call process.exit() at module load. That made it unimportable, and an unimportable scorer is an unmeasurable one — every suite in the repo tests the cascade's mechanism and none of them can say whether an agent would find the right concept in the first place. Move scoring, tokenization and snippeting into search.mjs as pure functions over an injected layer array. Behavior is unchanged: same occurrence counting, same field weights, same tie-breaks, same capture recency half-life. mcp-server.mjs keeps telemetry emission, since that is a server concern rather than a ranking one. This is the prerequisite for scoring the ranker against a golden question set; the scorer itself is not touched here. Signed-off-by: John Siracusa Co-authored-by: Claude Opus 5 --- packages/core/src/mcp-server.mjs | 105 +------------------- packages/core/src/search.mjs | 162 +++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 102 deletions(-) create mode 100644 packages/core/src/search.mjs diff --git a/packages/core/src/mcp-server.mjs b/packages/core/src/mcp-server.mjs index de1075a..6e85841 100755 --- a/packages/core/src/mcp-server.mjs +++ b/packages/core/src/mcp-server.mjs @@ -11,6 +11,7 @@ import path from "node:path"; import readline from "node:readline"; import { resolveConcept } from "./resolver.mjs"; +import { searchConcepts, searchCaptures } from "./search.mjs"; import { buildSources } from "./sources/index.mjs"; import { isTraversal } from "./sources/okf-local.mjs"; import { commitPaths, push } from "./sources/git-core.mjs"; @@ -363,47 +364,8 @@ async function callTool(name, toolArgs) { // ---- team-sync tools -------------------------------------------------------- -const DAY_MS = 86400000; - async function findCaptures({ query, kinds = null, limit = 10 }) { - if (!query || typeof query !== "string") throw new Error("find_captures requires a non-empty query string"); - const tokens = tokenize(query); - if (tokens.length === 0) throw new Error("find_captures query must contain at least one searchable token"); - - const rows = []; - for (const source of layers) { - for (const id of await source.listConceptIds()) { - if (!id.startsWith("captures/")) continue; - const entry = await source.loadConcept(id); - if (!entry) continue; - const { frontmatter, sections } = entry; - if (kinds && !kinds.includes(frontmatter.kind)) continue; - const sectionText = sections.map((s) => s.lines.join("\n")).join("\n"); - const base = scoreText(tokens, [id, frontmatter.title ?? "", "", "", sectionText]); - if (base <= 0) continue; - const capturedAt = frontmatter.captured ?? null; - const capturedTime = capturedAt ? new Date(capturedAt).getTime() : NaN; - // An unparseable `captured` must not poison scoring with NaN (which makes - // the sort unstable). Treat it as age 0 (freshest) — it still surfaces. - const ageDays = Number.isNaN(capturedTime) ? 0 : Math.max(0, (Date.now() - capturedTime) / DAY_MS); - rows.push({ - id, - title: frontmatter.title ?? null, - kind: frontmatter.kind ?? null, - author: frontmatter.author ?? null, - capturedAt, - ageDays: Math.round(ageDays * 10) / 10, - status: frontmatter.status ?? "unreviewed", - score: base * 2 ** (-ageDays / 7), // true 7-day half-life - snippet: makeSnippet(sectionText, tokens), - layer: source.name, - }); - } - } - - const result = rows - .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) - .slice(0, Number(limit) || 10); + const result = await searchCaptures(layers, { query, kinds, limit }); for (const row of result) emitTelemetry({ event: "search_hit", concept: row.id, layer: row.layer, captureKind: row.kind }); return result; } @@ -472,35 +434,7 @@ async function confirmCaptureTool({ token }) { // ---- tools ---------------------------------------------------------------- async function search({ query, limit = 10 }) { - if (!query || typeof query !== "string") throw new Error("search requires a non-empty query string"); - const tokens = tokenize(query); - if (tokens.length === 0) throw new Error("search query must contain at least one searchable token"); - - const byId = new Map(); - for (const source of layers) { - for (const id of await source.listConceptIds()) { - const entry = await source.loadConcept(id); - if (!entry) continue; - const { frontmatter, sections } = entry; - const sectionText = sections.map((s) => s.lines.join("\n")).join("\n"); - const score = scoreText(tokens, [id, frontmatter.title ?? "", frontmatter.description ?? "", String(frontmatter.tags ?? ""), sectionText]); - if (score <= 0) continue; - - const existing = byId.get(id); - if (!existing) { - byId.set(id, { id, title: frontmatter.title ?? null, score, layers: [source.name], snippet: makeSnippet(sectionText, tokens) }); - } else { - existing.score += score; - existing.layers.push(source.name); - if (!existing.title) existing.title = frontmatter.title ?? null; - } - } - } - - return [...byId.values()] - .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) - .slice(0, Number(limit) || 10) - .map((entry) => ({ ...entry, layers: orderLayerNames(entry.layers) })); + return await searchConcepts(layers, { query, limit }); } async function readFileTool({ concept_id, layer }) { @@ -685,39 +619,6 @@ function assembleMarkdown(resolved) { return front + bodyParts.join("\n\n"); } -function scoreText(tokens, haystacks) { - return tokens.reduce((total, token) => { - return total + haystacks.reduce((subtotal, haystack, index) => { - const weight = index <= 2 ? 4 : index === 3 ? 2 : 1; - return subtotal + countOccurrences(String(haystack).toLowerCase(), token) * weight; - }, 0); - }, 0); -} - -function tokenize(query) { - return query.toLowerCase().match(/[a-z0-9_-]+/g) ?? []; -} - -function countOccurrences(haystack, needle) { - if (!needle) return 0; - let count = 0; - let index = haystack.indexOf(needle); - while (index !== -1) { - count += 1; - index = haystack.indexOf(needle, index + needle.length); - } - return count; -} - -function makeSnippet(body, tokens) { - const lower = body.toLowerCase(); - const positions = tokens.map((token) => lower.indexOf(token)).filter((index) => index >= 0); - if (positions.length === 0) return body.trim().slice(0, 240); - const start = Math.max(0, Math.min(...positions) - 80); - const end = Math.min(body.length, start + 240); - return `${start > 0 ? "..." : ""}${body.slice(start, end).trim()}${end < body.length ? "..." : ""}`; -} - function stripDecoration(value) { return value.split("#")[0].split("?")[0].trim(); } diff --git a/packages/core/src/search.mjs b/packages/core/src/search.mjs new file mode 100644 index 0000000..1236f29 --- /dev/null +++ b/packages/core/src/search.mjs @@ -0,0 +1,162 @@ +// Retrieval over a cascade of layer sources: the ranking behind the `search` +// and `find_captures` MCP tools. +// +// This lives outside mcp-server.mjs on purpose. That module parses argv and can +// call process.exit() at load time, so it cannot be imported — which meant the +// ranking had no way to be measured. +// +// Not to be confused with tokenize.mjs, which is the BPE tokenizer used for +// context-budget accounting. The tokenizer here is a query analyzer. + +const DAY_MS = 86400000; + +// Field order is load-bearing: scorers weight by position. 0-2 are the +// identifying fields (id, title, description), 3 is tags, 4 is the body. +const WEIGHTS = [4, 4, 4, 2, 1]; + +export function tokenizeQuery(query) { + return query.toLowerCase().match(/[a-z0-9_-]+/g) ?? []; +} + +function requireTokens(query, tool) { + if (!query || typeof query !== "string") throw new Error(`${tool} requires a non-empty query string`); + const tokens = tokenizeQuery(query); + if (tokens.length === 0) throw new Error(`${tool} query must contain at least one searchable token`); + return tokens; +} + +// One I/O pass over the cascade. Callers score the returned array in memory, so +// adding a second scoring pass costs nothing on top of the walk. +async function collectDocuments(layers, { prefix = null } = {}) { + const docs = []; + for (const source of layers) { + for (const id of await source.listConceptIds()) { + if (prefix && !id.startsWith(prefix)) continue; + const entry = await source.loadConcept(id); + if (!entry) continue; + const { frontmatter, sections } = entry; + docs.push({ + id, + layer: source.name, + frontmatter, + body: sections.map((section) => section.lines.join("\n")).join("\n"), + }); + } + } + return docs; +} + +function conceptFields(doc) { + return [ + doc.id, + doc.frontmatter.title ?? "", + doc.frontmatter.description ?? "", + String(doc.frontmatter.tags ?? ""), + doc.body, + ]; +} + +function captureFields(doc) { + // Captures carry no description or tags; keeping the arity fixed keeps the + // positional weights aligned with concept scoring. + return [doc.id, doc.frontmatter.title ?? "", "", "", doc.body]; +} + +export function scoreFields(tokens, fields) { + return tokens.reduce((total, token) => { + return total + fields.reduce((subtotal, field, index) => { + return subtotal + countOccurrences(String(field).toLowerCase(), token) * WEIGHTS[index]; + }, 0); + }, 0); +} + +function countOccurrences(haystack, needle) { + if (!needle) return 0; + let count = 0; + let index = haystack.indexOf(needle); + while (index !== -1) { + count += 1; + index = haystack.indexOf(needle, index + needle.length); + } + return count; +} + +export function makeSnippet(body, tokens) { + const lower = body.toLowerCase(); + const positions = tokens.map((token) => lower.indexOf(token)).filter((index) => index >= 0); + if (positions.length === 0) return body.trim().slice(0, 240); + const start = Math.max(0, Math.min(...positions) - 80); + const end = Math.min(body.length, start + 240); + return `${start > 0 ? "..." : ""}${body.slice(start, end).trim()}${end < body.length ? "..." : ""}`; +} + +function layerOrderer(layers) { + const levelByName = new Map(layers.map((layer) => [layer.name, layer.level])); + return (names) => [...new Set(names)].sort((a, b) => (levelByName.get(b) ?? 0) - (levelByName.get(a) ?? 0)); +} + +export async function searchConcepts(layers, { query, limit = 10 }) { + const tokens = requireTokens(query, "search"); + const orderLayerNames = layerOrderer(layers); + const docs = await collectDocuments(layers); + + const byId = new Map(); + for (const doc of docs) { + const score = scoreFields(tokens, conceptFields(doc)); + if (score <= 0) continue; + + const existing = byId.get(doc.id); + if (!existing) { + byId.set(doc.id, { + id: doc.id, + title: doc.frontmatter.title ?? null, + score, + layers: [doc.layer], + snippet: makeSnippet(doc.body, tokens), + }); + } else { + existing.score += score; + existing.layers.push(doc.layer); + if (!existing.title) existing.title = doc.frontmatter.title ?? null; + } + } + + return [...byId.values()] + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) + .slice(0, Number(limit) || 10) + .map((entry) => ({ ...entry, layers: orderLayerNames(entry.layers) })); +} + +export async function searchCaptures(layers, { query, kinds = null, limit = 10, now = Date.now() }) { + const tokens = requireTokens(query, "find_captures"); + const docs = await collectDocuments(layers, { prefix: "captures/" }); + + const rows = []; + for (const doc of docs) { + if (kinds && !kinds.includes(doc.frontmatter.kind)) continue; + const base = scoreFields(tokens, captureFields(doc)); + if (base <= 0) continue; + + const capturedAt = doc.frontmatter.captured ?? null; + const capturedTime = capturedAt ? new Date(capturedAt).getTime() : NaN; + // An unparseable `captured` must not poison scoring with NaN (which makes + // the sort unstable). Treat it as age 0 (freshest) — it still surfaces. + const ageDays = Number.isNaN(capturedTime) ? 0 : Math.max(0, (now - capturedTime) / DAY_MS); + rows.push({ + id: doc.id, + title: doc.frontmatter.title ?? null, + kind: doc.frontmatter.kind ?? null, + author: doc.frontmatter.author ?? null, + capturedAt, + ageDays: Math.round(ageDays * 10) / 10, + status: doc.frontmatter.status ?? "unreviewed", + score: base * 2 ** (-ageDays / 7), // true 7-day half-life + snippet: makeSnippet(doc.body, tokens), + layer: doc.layer, + }); + } + + return rows + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) + .slice(0, Number(limit) || 10); +} From fc64db7ba52613a4c9e78f8c458e227cad3bd97b Mon Sep 17 00:00:00 2001 From: John Siracusa Date: Wed, 29 Jul 2026 23:34:45 -0400 Subject: [PATCH 2/4] test(core): add a retrieval eval and gate it in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a golden question set over a committed three-layer corpus, scored against search.mjs: recall@1, recall@5, MRR, and whether resolving the answer still surfaces the layers that disagree. The first run is not flattering. On 38 questions written the way someone would actually type them: recall@1 0.263 recall@5 0.500 mrr 0.348 conflict 1.000 Half the questions never surface the right document at all, and decisions/service-stack wins almost everything regardless of what was asked. Three causes, which the eval separates: - Substring rather than token matching. The query word "I" matches the "i" inside identifier, in, and with, so any natural question hands the longest document a large score for free. - No inverse document frequency: a word in every document counts as much as a word in one. - No length normalization: repetition beats precision. conflict is already 1.000, which is the honest read on the engine. The cascade and the per-section conflict surfacing work. Retrieval is the broken half, and it was broken badly enough to hide the working half. The runner is wired into `npm test`, so a ranking change that loses recall fails the build rather than being discovered by a user. Accepting a new number requires --record --label "", and superseded numbers stay in baseline.json's history so a deliberate trade stays legible. No scorer changes here — this commit only establishes the measurement. Signed-off-by: John Siracusa Co-authored-by: Claude Opus 5 --- package.json | 3 +- packages/core/eval/baseline.json | 9 + .../corpus/company/decisions/data-storage.md | 23 ++ .../corpus/company/decisions/service-stack.md | 23 ++ packages/core/eval/corpus/company/index.md | 18 ++ .../corpus/company/policies/access-control.md | 23 ++ .../corpus/company/policies/data-retention.md | 23 ++ .../company/policies/incident-response.md | 23 ++ .../company/policies/security-baseline.md | 23 ++ .../company/runbooks/deploy-standard.md | 23 ++ .../corpus/company/standards/api-design.md | 23 ++ .../corpus/company/standards/code-review.md | 23 ++ .../corpus/company/standards/observability.md | 23 ++ .../personal/decisions/service-stack.md | 11 + packages/core/eval/corpus/personal/index.md | 12 + .../eval/corpus/personal/notes/local-dev.md | 19 ++ .../corpus/personal/notes/query-cheatsheet.md | 19 ++ .../core/eval/corpus/personal/scratch/todo.md | 11 + .../corpus/team/decisions/data-storage.md | 15 ++ .../corpus/team/decisions/schema-evolution.md | 19 ++ .../corpus/team/decisions/service-stack.md | 15 ++ .../team/decisions/streaming-platform.md | 23 ++ .../corpus/team/gotchas/kafka-rebalance.md | 19 ++ .../eval/corpus/team/gotchas/spark-memory.md | 23 ++ packages/core/eval/corpus/team/index.md | 18 ++ .../eval/corpus/team/runbooks/backfill.md | 23 ++ .../corpus/team/runbooks/deploy-standard.md | 15 ++ .../corpus/team/runbooks/pipeline-oncall.md | 23 ++ .../corpus/team/standards/observability.md | 15 ++ packages/core/eval/manifest.json | 7 + packages/core/eval/questions.json | 237 ++++++++++++++++++ packages/core/eval/run.mjs | 206 +++++++++++++++ 32 files changed, 989 insertions(+), 1 deletion(-) create mode 100644 packages/core/eval/baseline.json create mode 100644 packages/core/eval/corpus/company/decisions/data-storage.md create mode 100644 packages/core/eval/corpus/company/decisions/service-stack.md create mode 100644 packages/core/eval/corpus/company/index.md create mode 100644 packages/core/eval/corpus/company/policies/access-control.md create mode 100644 packages/core/eval/corpus/company/policies/data-retention.md create mode 100644 packages/core/eval/corpus/company/policies/incident-response.md create mode 100644 packages/core/eval/corpus/company/policies/security-baseline.md create mode 100644 packages/core/eval/corpus/company/runbooks/deploy-standard.md create mode 100644 packages/core/eval/corpus/company/standards/api-design.md create mode 100644 packages/core/eval/corpus/company/standards/code-review.md create mode 100644 packages/core/eval/corpus/company/standards/observability.md create mode 100644 packages/core/eval/corpus/personal/decisions/service-stack.md create mode 100644 packages/core/eval/corpus/personal/index.md create mode 100644 packages/core/eval/corpus/personal/notes/local-dev.md create mode 100644 packages/core/eval/corpus/personal/notes/query-cheatsheet.md create mode 100644 packages/core/eval/corpus/personal/scratch/todo.md create mode 100644 packages/core/eval/corpus/team/decisions/data-storage.md create mode 100644 packages/core/eval/corpus/team/decisions/schema-evolution.md create mode 100644 packages/core/eval/corpus/team/decisions/service-stack.md create mode 100644 packages/core/eval/corpus/team/decisions/streaming-platform.md create mode 100644 packages/core/eval/corpus/team/gotchas/kafka-rebalance.md create mode 100644 packages/core/eval/corpus/team/gotchas/spark-memory.md create mode 100644 packages/core/eval/corpus/team/index.md create mode 100644 packages/core/eval/corpus/team/runbooks/backfill.md create mode 100644 packages/core/eval/corpus/team/runbooks/deploy-standard.md create mode 100644 packages/core/eval/corpus/team/runbooks/pipeline-oncall.md create mode 100644 packages/core/eval/corpus/team/standards/observability.md create mode 100644 packages/core/eval/manifest.json create mode 100644 packages/core/eval/questions.json create mode 100644 packages/core/eval/run.mjs diff --git a/package.json b/package.json index 369f9b5..1ffecad 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Federated team knowledge with cascading layer precedence — OKF-compatible, MCP-ready.", "type": "module", "scripts": { - "test": "bash packages/core/tests/smoke-test.sh && bash packages/core/tests/resolver-test.sh && bash packages/core/tests/source-test.sh && bash packages/core/tests/files-source-test.sh && bash packages/core/tests/github-source-test.sh && node --test packages/core/tests/manifest.test.mjs && bash packages/core/tests/profile-runtime-test.sh && bash packages/core/tests/pack-test.sh && bash packages/core/tests/git-sync-test.sh && bash packages/core/tests/capture-test.sh && bash packages/core/tests/team-sync-mcp-test.sh && bash packages/core/tests/playground-test.sh && bash packages/core/tests/service-test.sh && bash packages/core/tests/mcp-respawn-test.sh && bash packages/core/tests/setup-robustness-test.sh", + "test": "bash packages/core/tests/smoke-test.sh && bash packages/core/tests/resolver-test.sh && bash packages/core/tests/source-test.sh && bash packages/core/tests/files-source-test.sh && bash packages/core/tests/github-source-test.sh && node --test packages/core/tests/manifest.test.mjs && bash packages/core/tests/profile-runtime-test.sh && bash packages/core/tests/pack-test.sh && bash packages/core/tests/git-sync-test.sh && bash packages/core/tests/capture-test.sh && bash packages/core/tests/team-sync-mcp-test.sh && bash packages/core/tests/playground-test.sh && bash packages/core/tests/service-test.sh && bash packages/core/tests/mcp-respawn-test.sh && bash packages/core/tests/setup-robustness-test.sh && npm run eval", + "eval": "node packages/core/eval/run.mjs", "mcp": "node mcp-server.mjs", "playground": "node apps/playground/server.mjs", "console:live": "npm --prefix apps/console run build:live && node apps/playground/server.mjs --console apps/console/dist", diff --git a/packages/core/eval/baseline.json b/packages/core/eval/baseline.json new file mode 100644 index 0000000..bf3f913 --- /dev/null +++ b/packages/core/eval/baseline.json @@ -0,0 +1,9 @@ +{ + "label": "substring occurrence counting (pre-BM25)", + "questions": 38, + "recall@1": 0.263, + "recall@5": 0.5, + "mrr": 0.348, + "conflict": 1, + "history": [] +} diff --git a/packages/core/eval/corpus/company/decisions/data-storage.md b/packages/core/eval/corpus/company/decisions/data-storage.md new file mode 100644 index 0000000..7abecb3 --- /dev/null +++ b/packages/core/eval/corpus/company/decisions/data-storage.md @@ -0,0 +1,23 @@ +--- +type: decision +title: Primary datastore standard +description: Which database a service should use for transactional state. +tags: [database, storage, standard] +updated: 2026-05-20 +--- + +## Transactional Store {#transactional} + +PostgreSQL 16 is the default database for service state. Managed instances are provisioned through the platform portal; teams do not run their own. + +## Caching {#caching} + +Redis for ephemeral cache only. Nothing that cannot be rebuilt from the primary datastore may live in Redis. + +## Analytics {#analytics} + +Analytical queries run against the warehouse, never against a service's production database. Read replicas exist for operational debugging, not reporting. + +## Migrations {#migrations} + +Schema changes ship as forward-only migrations reviewed by the owning team. Destructive column drops require a two-release deprecation window. diff --git a/packages/core/eval/corpus/company/decisions/service-stack.md b/packages/core/eval/corpus/company/decisions/service-stack.md new file mode 100644 index 0000000..2c0ac47 --- /dev/null +++ b/packages/core/eval/corpus/company/decisions/service-stack.md @@ -0,0 +1,23 @@ +--- +type: decision +title: Service stack standard +description: The org-wide default language, framework, and runtime for new services. +tags: [platform, language, standard] +updated: 2026-06-01 +--- + +## Language and Framework {#language} + +Spring Boot with Java 21. This is the standard for all new services org-wide. Teams that need a different runtime must file an exemption with the platform group and re-confirm it annually. + +## Build and Packaging {#build} + +Gradle with the shared convention plugin. Services publish an OCI image to the internal registry; no team maintains its own base image. + +## Runtime Targets {#runtime} + +Services run on the shared Kubernetes platform. Bare EC2 and per-team clusters were retired in 2025. + +## Enforcement {#enforcement} + +New services must pass the Java 21 conformance check in company CI. Existing exemptions expire twelve months after they are granted. diff --git a/packages/core/eval/corpus/company/index.md b/packages/core/eval/corpus/company/index.md new file mode 100644 index 0000000..a104c65 --- /dev/null +++ b/packages/core/eval/corpus/company/index.md @@ -0,0 +1,18 @@ +--- +type: index +title: Company knowledge +description: Organization-wide canonical standards, policies, and runbooks. +--- + +# Company layer + +- [Service stack standard](decisions/service-stack.md) +- [Primary datastore standard](decisions/data-storage.md) +- [Security baseline](policies/security-baseline.md) +- [Data retention policy](policies/data-retention.md) +- [Incident response](policies/incident-response.md) +- [Access control](policies/access-control.md) +- [Standard deployment process](runbooks/deploy-standard.md) +- [Code review expectations](standards/code-review.md) +- [Observability standard](standards/observability.md) +- [API design conventions](standards/api-design.md) diff --git a/packages/core/eval/corpus/company/policies/access-control.md b/packages/core/eval/corpus/company/policies/access-control.md new file mode 100644 index 0000000..f534a13 --- /dev/null +++ b/packages/core/eval/corpus/company/policies/access-control.md @@ -0,0 +1,23 @@ +--- +type: policy +title: Access control +description: How repository, cluster, and data access is granted and reviewed. +tags: [security, permissions, access] +updated: 2026-02-11 +--- + +## Repository Access {#repos} + +Repository membership is the access-control boundary. Grant the narrowest team that needs the code; personal collaborator grants are not permitted on service repositories. + +## Production Access {#production} + +Production cluster access is time-boxed and requested through the access portal. Standing production credentials do not exist for engineers. + +## Warehouse Access {#warehouse} + +Warehouse datasets carrying customer identifiers are restricted. Access requires a named business justification and lapses after ninety days without renewal. + +## Review Cadence {#review} + +Access grants are reviewed quarterly. An unreviewed grant is revoked automatically rather than extended by default. diff --git a/packages/core/eval/corpus/company/policies/data-retention.md b/packages/core/eval/corpus/company/policies/data-retention.md new file mode 100644 index 0000000..df4b922 --- /dev/null +++ b/packages/core/eval/corpus/company/policies/data-retention.md @@ -0,0 +1,23 @@ +--- +type: policy +title: Data retention policy +description: How long each class of record may be kept before deletion. +tags: [compliance, retention, privacy] +updated: 2026-04-02 +--- + +## Customer Records {#customer} + +Customer account records are retained for seven years after account closure, then hard-deleted. Deletion is verified by the annual compliance audit. + +## Event and Telemetry Data {#telemetry} + +Raw event streams are retained for ninety days. Aggregated metrics derived from them may be kept indefinitely provided they carry no user identifier. + +## Logs {#logs} + +Application logs are retained for thirty days in hot storage and one year in cold storage. Logs must never contain personally identifiable information. + +## Deletion Requests {#deletion} + +A verified user deletion request must be honored across every system within thirty days, including warehouse copies and backups scheduled for expiry. diff --git a/packages/core/eval/corpus/company/policies/incident-response.md b/packages/core/eval/corpus/company/policies/incident-response.md new file mode 100644 index 0000000..9592be6 --- /dev/null +++ b/packages/core/eval/corpus/company/policies/incident-response.md @@ -0,0 +1,23 @@ +--- +type: policy +title: Incident response +description: Severity levels, paging expectations, and postmortem requirements. +tags: [oncall, incident, sev, postmortem] +updated: 2026-03-18 +--- + +## Severity Levels {#severity} + +Sev1 is customer-visible loss of a core flow. Sev2 is degraded service or a workaround-available failure. Sev3 is internal-only impact. Anything below that is a ticket, not an incident. + +## Paging {#paging} + +Sev1 pages the owning team immediately and the incident commander rota after ten minutes. Sev2 pages during business hours only. Nobody is paged for Sev3. + +## Communication {#communication} + +The incident channel is the single source of truth during an active incident. Status updates go out every thirty minutes for Sev1 until mitigation. + +## Postmortems {#postmortem} + +Every Sev1 and Sev2 gets a written, blameless postmortem within five business days. Action items are tracked to completion; a postmortem with no owner on an action item is not complete. diff --git a/packages/core/eval/corpus/company/policies/security-baseline.md b/packages/core/eval/corpus/company/policies/security-baseline.md new file mode 100644 index 0000000..1b42218 --- /dev/null +++ b/packages/core/eval/corpus/company/policies/security-baseline.md @@ -0,0 +1,23 @@ +--- +type: policy +title: Security baseline +description: Minimum security controls every service must meet before it serves traffic. +tags: [security, secrets, encryption, compliance] +updated: 2026-06-10 +--- + +## Secrets Handling {#secrets} + +Services read secrets from the company vault at startup. No credential may be committed to a repository, baked into an image, or passed as a plaintext environment variable in a deployment manifest. + +## Encryption {#encryption} + +Personally identifiable information is encrypted at rest and in transit. TLS 1.3 is required for all internal and external traffic; TLS 1.2 is permitted only for third-party endpoints that cannot negotiate 1.3. + +## Authentication {#authentication} + +Service-to-service calls authenticate through company SSO with short-lived tokens. Long-lived API keys are prohibited except for vendor integrations that offer no alternative, and those must be rotated quarterly. + +## Dependency Hygiene {#dependencies} + +Automated dependency updates are mandatory. A critical advisory must be patched within seven days; a high advisory within thirty. diff --git a/packages/core/eval/corpus/company/runbooks/deploy-standard.md b/packages/core/eval/corpus/company/runbooks/deploy-standard.md new file mode 100644 index 0000000..d3f104e --- /dev/null +++ b/packages/core/eval/corpus/company/runbooks/deploy-standard.md @@ -0,0 +1,23 @@ +--- +type: runbook +title: Standard deployment process +description: How a change gets from a merged pull request to production. +tags: [deploy, ci, release, rollback] +updated: 2026-05-05 +--- + +## Pipeline {#pipeline} + +A merge to the default branch builds an image, runs the full test suite, and deploys to staging automatically. Production is a manual promotion from a green staging build. + +## Promotion Gates {#gates} + +Promotion requires a green staging run, no open Sev1, and an approver who is not the change author. + +## Rollback {#rollback} + +Roll back by promoting the previous known-good image tag. Do not roll forward with a hotfix during an active incident unless the previous image is also broken. + +## Deployment Windows {#windows} + +Deploys are allowed any weekday before 16:00 local time for the owning team. Friday afternoon and holiday-eve deploys require an explicit exception. diff --git a/packages/core/eval/corpus/company/standards/api-design.md b/packages/core/eval/corpus/company/standards/api-design.md new file mode 100644 index 0000000..240bf4e --- /dev/null +++ b/packages/core/eval/corpus/company/standards/api-design.md @@ -0,0 +1,23 @@ +--- +type: standard +title: API design conventions +description: Naming, versioning, and error conventions for HTTP services. +tags: [api, http, rest, versioning] +updated: 2026-03-01 +--- + +## Resource Naming {#naming} + +Paths are plural nouns in kebab-case. Verbs belong in the HTTP method, not the path. + +## Versioning {#versioning} + +Breaking changes ship behind a new major version in the path. A major version is supported for twelve months after its successor is generally available. + +## Errors {#errors} + +Errors return a machine-readable code alongside a human-readable message. Never return a 200 with an error body. + +## Pagination {#pagination} + +Collection endpoints paginate by opaque cursor. Offset pagination is not permitted on datasets that can change under the reader. diff --git a/packages/core/eval/corpus/company/standards/code-review.md b/packages/core/eval/corpus/company/standards/code-review.md new file mode 100644 index 0000000..6d4ad89 --- /dev/null +++ b/packages/core/eval/corpus/company/standards/code-review.md @@ -0,0 +1,23 @@ +--- +type: standard +title: Code review expectations +description: What a reviewer is accountable for and how long review should take. +tags: [review, quality, process] +updated: 2026-01-22 +--- + +## Required Approvals {#approvals} + +Every change to a service repository needs one approving review from someone outside the change's authorship. Two approvals are required for changes to authentication, billing, or data deletion paths. + +## Reviewer Responsibility {#responsibility} + +A reviewer is accountable for correctness, test coverage, and whether the change matches the stated intent. A reviewer is not accountable for style; formatters handle that. + +## Turnaround {#turnaround} + +First response within one business day. A review request older than two business days may be escalated to the team lead. + +## Automation {#automation} + +Formatting, linting, and dependency policy are enforced by CI, not by humans in review comments. diff --git a/packages/core/eval/corpus/company/standards/observability.md b/packages/core/eval/corpus/company/standards/observability.md new file mode 100644 index 0000000..a1dec86 --- /dev/null +++ b/packages/core/eval/corpus/company/standards/observability.md @@ -0,0 +1,23 @@ +--- +type: standard +title: Observability standard +description: Required logging, metrics, and tracing for every production service. +tags: [logging, metrics, tracing, monitoring, alerts] +updated: 2026-04-28 +--- + +## Structured Logging {#logging} + +Logs are emitted as structured JSON with a request identifier on every line. Free-text logging without a correlation field is not acceptable in production code. + +## Metrics {#metrics} + +Every service exports request rate, error rate, and latency percentiles. Dashboards are generated from the shared template rather than hand-built per team. + +## Tracing {#tracing} + +Distributed tracing is enabled by default with head-based sampling at one percent. Raise the sample rate temporarily during an investigation, then put it back. + +## Alerting {#alerting} + +Alerts fire on symptoms customers feel, not on causes. An alert that nobody acts on is deleted rather than muted. diff --git a/packages/core/eval/corpus/personal/decisions/service-stack.md b/packages/core/eval/corpus/personal/decisions/service-stack.md new file mode 100644 index 0000000..4680284 --- /dev/null +++ b/packages/core/eval/corpus/personal/decisions/service-stack.md @@ -0,0 +1,11 @@ +--- +type: decision +title: Service stack standard +description: My standing exemption note for the reconciliation service. +tags: [language, exemption] +updated: 2026-07-10 +--- + +## Language and Framework {#language} + +The reconciliation service I own is Python 3.12, not Scala and not Java. It was inherited from the finance team in the 2025 reorg and the rewrite is not funded. The exemption is on file with the platform group and expires 2027-01-01. diff --git a/packages/core/eval/corpus/personal/index.md b/packages/core/eval/corpus/personal/index.md new file mode 100644 index 0000000..fcdd86d --- /dev/null +++ b/packages/core/eval/corpus/personal/index.md @@ -0,0 +1,12 @@ +--- +type: index +title: Personal knowledge +description: Local notes, cheatsheets, and personal overrides. +--- + +# Personal layer + +- [Service stack standard](decisions/service-stack.md) +- [Local development setup](notes/local-dev.md) +- [ClickHouse query cheatsheet](notes/query-cheatsheet.md) +- [Scratch todo](scratch/todo.md) diff --git a/packages/core/eval/corpus/personal/notes/local-dev.md b/packages/core/eval/corpus/personal/notes/local-dev.md new file mode 100644 index 0000000..4ecef16 --- /dev/null +++ b/packages/core/eval/corpus/personal/notes/local-dev.md @@ -0,0 +1,19 @@ +--- +type: note +title: Local development setup +description: Getting a pipeline running on a laptop without a cluster. +tags: [local, development, docker, sbt] +updated: 2026-07-12 +--- + +## Running Locally {#running} + +`docker compose up` in the platform repo brings up Kafka, the schema registry, and a single-node Postgres. Spark runs in local mode against those; there is no local ClickHouse, so analytics assertions have to be stubbed. + +## Common Snag {#snag} + +The compose file binds Kafka on the host network, so the advertised listener has to be `localhost:9092` or the driver connects to the broker and then fails on the second hop with a name it cannot resolve. This costs everyone an afternoon exactly once. + +## Speeding Up Iteration {#iteration} + +Keep an sbt shell open and use `~testQuick`. A cold sbt start is slower than the test run it is about to do. diff --git a/packages/core/eval/corpus/personal/notes/query-cheatsheet.md b/packages/core/eval/corpus/personal/notes/query-cheatsheet.md new file mode 100644 index 0000000..3ace36e --- /dev/null +++ b/packages/core/eval/corpus/personal/notes/query-cheatsheet.md @@ -0,0 +1,19 @@ +--- +type: note +title: ClickHouse query cheatsheet +description: Query shapes I keep forgetting. +tags: [clickhouse, sql, queries] +updated: 2026-06-27 +--- + +## Time Buckets {#buckets} + +`toStartOfInterval(ts, INTERVAL 5 MINUTE)` for bucketing. `toStartOfFiveMinute` also exists but does not generalize, so I stopped using it. + +## Approximate Counts {#approx} + +`uniqCombined` is close enough for dashboards and dramatically cheaper than `uniqExact`. Use the exact variant only when someone is going to reconcile the number against finance. + +## Debugging A Slow Query {#slow} + +`EXPLAIN indexes = 1` shows whether the primary index was used. Nine times out of ten a slow query is a filter that does not match the sort key prefix. diff --git a/packages/core/eval/corpus/personal/scratch/todo.md b/packages/core/eval/corpus/personal/scratch/todo.md new file mode 100644 index 0000000..4862d5e --- /dev/null +++ b/packages/core/eval/corpus/personal/scratch/todo.md @@ -0,0 +1,11 @@ +--- +type: note +title: Scratch todo +description: Personal working list, not team knowledge. +tags: [scratch] +updated: 2026-07-14 +--- + +## This Week {#week} + +Finish the reconciliation exemption renewal paperwork. Ask about the ClickHouse retention change. Follow up on the rebalance fix landing in staging. diff --git a/packages/core/eval/corpus/team/decisions/data-storage.md b/packages/core/eval/corpus/team/decisions/data-storage.md new file mode 100644 index 0000000..c98161c --- /dev/null +++ b/packages/core/eval/corpus/team/decisions/data-storage.md @@ -0,0 +1,15 @@ +--- +type: decision +title: Primary datastore standard +description: Where the data platform team keeps state, including the analytics exception. +tags: [database, clickhouse, storage] +updated: 2026-06-22 +--- + +## Transactional Store {#transactional} + +PostgreSQL 16 for pipeline metadata and job bookkeeping, matching the org default. Nothing controversial here. + +## Analytics {#analytics} + +ClickHouse is our analytical store, not the central warehouse. Interactive queries over the event stream need sub-second response and the warehouse cannot do that at our cardinality. The warehouse remains the system of record for finance-facing reporting. diff --git a/packages/core/eval/corpus/team/decisions/schema-evolution.md b/packages/core/eval/corpus/team/decisions/schema-evolution.md new file mode 100644 index 0000000..ddc0b7a --- /dev/null +++ b/packages/core/eval/corpus/team/decisions/schema-evolution.md @@ -0,0 +1,19 @@ +--- +type: decision +title: Schema evolution rules +description: Which Avro schema changes are allowed without breaking downstream consumers. +tags: [avro, schema, compatibility] +updated: 2026-05-29 +--- + +## Compatibility Mode {#compatibility} + +The registry enforces backward compatibility. A new schema must be readable by consumers still on the previous version. + +## Allowed Changes {#allowed} + +Adding a field with a default is allowed. Widening a numeric type is allowed. Adding a new enum symbol is allowed only if consumers use a default branch. + +## Forbidden Changes {#forbidden} + +Removing a field, renaming a field, and narrowing a type all break readers and are rejected at registration. Renaming is the one people try most often; ship an additive field and deprecate the old one instead. diff --git a/packages/core/eval/corpus/team/decisions/service-stack.md b/packages/core/eval/corpus/team/decisions/service-stack.md new file mode 100644 index 0000000..ac75baa --- /dev/null +++ b/packages/core/eval/corpus/team/decisions/service-stack.md @@ -0,0 +1,15 @@ +--- +type: decision +title: Service stack standard +description: What the data platform team actually runs, and why it differs from the org default. +tags: [platform, language, scala, spark] +updated: 2026-06-18 +--- + +## Language and Framework {#language} + +Scala 2.13 with Spark Structured Streaming for pipelines, and Java 17 for the remaining legacy request/response services. We do not use Spring Boot: our workloads are streaming and batch, not request/response, and the Spring lifecycle fights the Spark driver model. + +## Build and Packaging {#build} + +sbt with the assembly plugin, because the shared Gradle convention plugin has no Scala cross-build support. Images are still published to the internal registry. diff --git a/packages/core/eval/corpus/team/decisions/streaming-platform.md b/packages/core/eval/corpus/team/decisions/streaming-platform.md new file mode 100644 index 0000000..b6fee74 --- /dev/null +++ b/packages/core/eval/corpus/team/decisions/streaming-platform.md @@ -0,0 +1,23 @@ +--- +type: decision +title: Streaming platform +description: Kafka topic conventions, partitioning, and delivery guarantees for the pipeline fleet. +tags: [kafka, streaming, topics, partitions] +updated: 2026-06-30 +--- + +## Broker Platform {#broker} + +Managed Kafka, three brokers per environment. We evaluated Pulsar in 2025 and stayed on Kafka because the operational tooling and our existing consumer code were both already there. + +## Topic Naming {#topics} + +Topics are named `...v`. A schema-breaking change gets a new major and a parallel topic; consumers migrate and the old topic is retired after thirty days. + +## Partitioning {#partitioning} + +Partition by entity identifier so per-entity ordering holds. Do not partition by timestamp: it produces hot partitions during backfills and destroys ordering guarantees the downstream joins rely on. + +## Delivery Semantics {#delivery} + +At-least-once delivery with idempotent writes downstream. Exactly-once is available in the Kafka transactional API but we do not use it; the throughput cost is real and every consumer we own is already idempotent. diff --git a/packages/core/eval/corpus/team/gotchas/kafka-rebalance.md b/packages/core/eval/corpus/team/gotchas/kafka-rebalance.md new file mode 100644 index 0000000..f33f847 --- /dev/null +++ b/packages/core/eval/corpus/team/gotchas/kafka-rebalance.md @@ -0,0 +1,19 @@ +--- +type: gotcha +title: Consumer group rebalance storms +description: Why consumers thrash on deploy and how to stop the repeated rebalancing. +tags: [kafka, rebalance, consumer, timeout] +updated: 2026-06-19 +--- + +## Symptom {#symptom} + +After a deploy, the consumer group rebalances continuously. Throughput collapses and lag climbs even though every instance looks healthy. + +## Usual Cause {#cause} + +`max.poll.interval.ms` is shorter than the slowest batch takes to process. The coordinator decides the member is dead, revokes its partitions, and the member rejoins mid-batch, which triggers the next rebalance. + +## Fix {#fix} + +Raise `max.poll.interval.ms` above the worst-case batch duration, or lower `max.poll.records` so a batch finishes inside the interval. Use cooperative sticky assignment so a rejoin does not stop every other consumer. diff --git a/packages/core/eval/corpus/team/gotchas/spark-memory.md b/packages/core/eval/corpus/team/gotchas/spark-memory.md new file mode 100644 index 0000000..372f59d --- /dev/null +++ b/packages/core/eval/corpus/team/gotchas/spark-memory.md @@ -0,0 +1,23 @@ +--- +type: gotcha +title: Spark executor out-of-memory failures +description: Why executors die during joins and shuffles, and what actually fixes it. +tags: [spark, memory, oom, shuffle, skew] +updated: 2026-07-05 +--- + +## Symptom {#symptom} + +Executors are killed mid-stage with exit code 137 and the stage retries until the job fails. The driver log blames the container, which is misleading: the container was killed by the node, not by Spark. + +## Usual Cause {#cause} + +Data skew in a join key. One partition holds most of the rows, that single executor exceeds its container limit, and the node reaps it. Raising executor memory moves the failure later without preventing it. + +## Fix {#fix} + +Salt the skewed key or enable adaptive query execution with skew join handling. Repartitioning before the join helps only when the skew is mild. + +## What Does Not Help {#not} + +Increasing `spark.executor.memory` past the node's allocatable ceiling causes the scheduler to refuse the request instead, which reads as a hang. Disabling off-heap does nothing for a skew problem. diff --git a/packages/core/eval/corpus/team/index.md b/packages/core/eval/corpus/team/index.md new file mode 100644 index 0000000..dd4935a --- /dev/null +++ b/packages/core/eval/corpus/team/index.md @@ -0,0 +1,18 @@ +--- +type: index +title: Data platform team knowledge +description: How the data platform team works, including deliberate departures from the org standard. +--- + +# Team layer (Data Platform) + +- [Service stack standard](decisions/service-stack.md) +- [Primary datastore standard](decisions/data-storage.md) +- [Streaming platform](decisions/streaming-platform.md) +- [Schema evolution rules](decisions/schema-evolution.md) +- [Pipeline on-call](runbooks/pipeline-oncall.md) +- [Running a backfill](runbooks/backfill.md) +- [Standard deployment process](runbooks/deploy-standard.md) +- [Observability standard](standards/observability.md) +- [Spark executor out-of-memory failures](gotchas/spark-memory.md) +- [Consumer group rebalance storms](gotchas/kafka-rebalance.md) diff --git a/packages/core/eval/corpus/team/runbooks/backfill.md b/packages/core/eval/corpus/team/runbooks/backfill.md new file mode 100644 index 0000000..90d0e2c --- /dev/null +++ b/packages/core/eval/corpus/team/runbooks/backfill.md @@ -0,0 +1,23 @@ +--- +type: runbook +title: Running a backfill +description: How to reprocess a historical window without taking down the live pipeline. +tags: [backfill, reprocessing, pipeline] +updated: 2026-06-14 +--- + +## Before You Start {#before} + +Backfills compete with the live job for cluster capacity and for Kafka broker throughput. Announce the window in the team channel and run outside the daily peak. + +## Isolation {#isolation} + +Run the backfill as a separate application with its own consumer group and its own checkpoint directory. Never reuse the live job's checkpoint: reusing it rewinds production. + +## Rate Limiting {#rate} + +Cap the backfill with `maxOffsetsPerTrigger`. An uncapped backfill reads the entire retention window in one micro-batch, blows the executor heap, and pages whoever is on call. + +## Verification {#verify} + +Compare row counts and a checksum for the reprocessed window against the source before deleting the old partition. A backfill that silently drops rows looks identical to a successful one until someone asks about a number three weeks later. diff --git a/packages/core/eval/corpus/team/runbooks/deploy-standard.md b/packages/core/eval/corpus/team/runbooks/deploy-standard.md new file mode 100644 index 0000000..3fdbace --- /dev/null +++ b/packages/core/eval/corpus/team/runbooks/deploy-standard.md @@ -0,0 +1,15 @@ +--- +type: runbook +title: Standard deployment process +description: How the data platform team ships pipeline changes, which differs from the org promotion model. +tags: [deploy, release, pipeline] +updated: 2026-06-25 +--- + +## Pipeline {#pipeline} + +Pipeline jobs deploy on drain, not on promotion. A merge builds the image; the running job finishes its current micro-batch, checkpoints, and exits, and the scheduler starts the new version from the same checkpoint. + +## Deployment Windows {#windows} + +No pipeline deploys during the nightly aggregation window, 01:00 to 05:00 UTC. Outside that, any weekday is fine, including Friday afternoon: a drain-and-restart is not a risky operation for us the way a request-serving rollout is. diff --git a/packages/core/eval/corpus/team/runbooks/pipeline-oncall.md b/packages/core/eval/corpus/team/runbooks/pipeline-oncall.md new file mode 100644 index 0000000..978b60c --- /dev/null +++ b/packages/core/eval/corpus/team/runbooks/pipeline-oncall.md @@ -0,0 +1,23 @@ +--- +type: runbook +title: Pipeline on-call +description: First response when a streaming job stalls, lags, or dies overnight. +tags: [oncall, pipeline, lag, spark, kafka] +updated: 2026-07-02 +--- + +## Triage Order {#triage} + +Check consumer lag first, then the driver logs, then upstream producer health. Most overnight pages are a lag alert caused by an upstream producer burst, not by anything wrong in our job. + +## Stalled Job {#stalled} + +A job that shows zero progress for more than ten minutes is stalled. Restart the driver before investigating: a stalled Structured Streaming query almost never recovers on its own and the checkpoint makes the restart safe. + +## Growing Lag {#lag} + +If lag grows steadily but the job is healthy, the job is under-provisioned rather than broken. Raise executor count first; raising memory per executor rarely helps a throughput problem. + +## When To Escalate {#escalate} + +Escalate to the platform team if brokers are unreachable or the schema registry is down. Do not escalate for a single failed micro-batch; the retry handles it. diff --git a/packages/core/eval/corpus/team/standards/observability.md b/packages/core/eval/corpus/team/standards/observability.md new file mode 100644 index 0000000..5bfdfee --- /dev/null +++ b/packages/core/eval/corpus/team/standards/observability.md @@ -0,0 +1,15 @@ +--- +type: standard +title: Observability standard +description: What a streaming job must emit, on top of the org baseline. +tags: [monitoring, metrics, lag, alerts] +updated: 2026-06-08 +--- + +## Metrics {#metrics} + +Request rate and latency percentiles are meaningless for a batch job. Streaming jobs export consumer lag, micro-batch duration, input rows per trigger, and checkpoint age instead. + +## Alerting {#alerting} + +Alert on consumer lag exceeding fifteen minutes and on checkpoint age exceeding three trigger intervals. Do not alert on individual task failures; Spark retries them and the noise trains people to ignore the channel. diff --git a/packages/core/eval/manifest.json b/packages/core/eval/manifest.json new file mode 100644 index 0000000..0a91527 --- /dev/null +++ b/packages/core/eval/manifest.json @@ -0,0 +1,7 @@ +{ + "layers": [ + { "name": "personal", "level": 3, "path": "corpus/personal" }, + { "name": "team", "level": 2, "path": "corpus/team" }, + { "name": "company", "level": 0, "path": "corpus/company" } + ] +} diff --git a/packages/core/eval/questions.json b/packages/core/eval/questions.json new file mode 100644 index 0000000..37ccb8d --- /dev/null +++ b/packages/core/eval/questions.json @@ -0,0 +1,237 @@ +{ + "readme": "Golden question set for retrieval. `expect` lists the concept ids that answer the question — a hit on any of them counts, so a question with two legitimate answers is not penalized for picking either. `conflict` asserts that resolving the concept surfaces dissent from the named layers, which is the thing ContextCake claims to do that a flat wiki cannot. `probes` records why the question is in the set; when a question starts passing or failing, that note is what tells you whether the change was real.", + "questions": [ + { + "id": "q01", + "question": "what language should I use for a new service?", + "expect": ["decisions/service-stack"], + "conflict": { "concept": "decisions/service-stack", "layers": ["team", "company"] }, + "probes": "natural phrasing with stopwords; three layers disagree and all three must survive" + }, + { + "id": "q02", + "question": "which database do we use for service state?", + "expect": ["decisions/data-storage"], + "probes": "direct lexical match on a distinctive term" + }, + { + "id": "q03", + "question": "what databases are we allowed to use?", + "expect": ["decisions/data-storage"], + "probes": "plural form of a term the corpus writes in the singular" + }, + { + "id": "q04", + "question": "where should analytics queries run?", + "expect": ["decisions/data-storage"], + "conflict": { "concept": "decisions/data-storage", "layers": ["company"] }, + "probes": "team overrides the org answer; the org position must still be visible" + }, + { + "id": "q05", + "question": "how do I store secrets?", + "expect": ["policies/security-baseline"], + "probes": "short natural question, single strong term" + }, + { + "id": "q06", + "question": "what TLS version is required?", + "expect": ["policies/security-baseline"], + "probes": "acronym plus a common noun" + }, + { + "id": "q07", + "question": "how long do we keep logs?", + "expect": ["policies/data-retention"], + "probes": "the doc says retained, the question says keep — no lexical overlap on the verb" + }, + { + "id": "q08", + "question": "how long are customer records retained?", + "expect": ["policies/data-retention"], + "probes": "same concept as q07 with the document's own vocabulary, to isolate vocabulary effects" + }, + { + "id": "q09", + "question": "when does a Sev1 page someone?", + "expect": ["policies/incident-response"], + "probes": "distinctive token that appears in exactly one document" + }, + { + "id": "q10", + "question": "do we need a postmortem for a Sev2?", + "expect": ["policies/incident-response"], + "probes": "two distinctive tokens, both in the same document" + }, + { + "id": "q11", + "question": "how do I get production access?", + "expect": ["policies/access-control"], + "probes": "production also appears in deploy, observability and security docs" + }, + { + "id": "q12", + "question": "how many approvals does a pull request need?", + "expect": ["standards/code-review"], + "probes": "the corpus says change and review, never pull request" + }, + { + "id": "q13", + "question": "who reviews changes to billing code?", + "expect": ["standards/code-review"], + "probes": "reviews vs review — morphological variant of the key term" + }, + { + "id": "q14", + "question": "how do I roll back a bad deploy?", + "expect": ["runbooks/deploy-standard"], + "probes": "roll back vs rollback — tokenization boundary" + }, + { + "id": "q15", + "question": "can I deploy on a Friday afternoon?", + "expect": ["runbooks/deploy-standard"], + "conflict": { "concept": "runbooks/deploy-standard", "layers": ["company"] }, + "probes": "team explicitly reverses the org rule; both answers must be present" + }, + { + "id": "q16", + "question": "what are the deployment windows?", + "expect": ["runbooks/deploy-standard"], + "probes": "deployment vs deploys — morphological variant" + }, + { + "id": "q17", + "question": "what metrics should my service export?", + "expect": ["standards/observability"], + "conflict": { "concept": "standards/observability", "layers": ["company"] }, + "probes": "streaming metrics override request metrics; the org list must still show" + }, + { + "id": "q18", + "question": "how should I paginate a collection endpoint?", + "expect": ["standards/api-design"], + "probes": "paginate vs pagination — morphological variant" + }, + { + "id": "q19", + "question": "how do we version breaking API changes?", + "expect": ["standards/api-design"], + "probes": "version and breaking both appear in schema-evolution as a distractor" + }, + { + "id": "q20", + "question": "why are my Spark executors getting killed?", + "expect": ["gotchas/spark-memory"], + "probes": "executors vs executor; killed appears in the target document" + }, + { + "id": "q21", + "question": "executor exit code 137 during a join", + "expect": ["gotchas/spark-memory"], + "probes": "keyword-style query pasted from an error, no natural language" + }, + { + "id": "q22", + "question": "how do we handle skewed join keys?", + "expect": ["gotchas/spark-memory"], + "probes": "skewed vs skew — morphological variant on the decisive term" + }, + { + "id": "q23", + "question": "my consumer group keeps rebalancing after a deploy", + "expect": ["gotchas/kafka-rebalance"], + "probes": "rebalancing vs rebalance; deploy is a strong distractor term" + }, + { + "id": "q24", + "question": "how do I stop rebalance storms?", + "expect": ["gotchas/kafka-rebalance"], + "probes": "exact title vocabulary — should be the easiest question in the set" + }, + { + "id": "q25", + "question": "how should Kafka topics be named?", + "expect": ["decisions/streaming-platform"], + "probes": "named vs naming/names — morphological variant" + }, + { + "id": "q26", + "question": "should I partition by timestamp?", + "expect": ["decisions/streaming-platform"], + "probes": "the answer is a warning against the thing asked about" + }, + { + "id": "q27", + "question": "do we use exactly-once delivery?", + "expect": ["decisions/streaming-platform"], + "probes": "hyphenated term; tokenizer keeps hyphens so exactly-once splits differently than expected" + }, + { + "id": "q28", + "question": "can I rename a field in an Avro schema?", + "expect": ["decisions/schema-evolution"], + "probes": "rename vs renaming; field appears across many documents" + }, + { + "id": "q29", + "question": "which schema changes are backward compatible?", + "expect": ["decisions/schema-evolution"], + "probes": "compatible vs compatibility — morphological variant" + }, + { + "id": "q30", + "question": "the pipeline is stalled, what do I do?", + "expect": ["runbooks/pipeline-oncall"], + "probes": "heavy stopword load around two content words" + }, + { + "id": "q31", + "question": "consumer lag is growing overnight", + "expect": ["runbooks/pipeline-oncall", "gotchas/kafka-rebalance"], + "probes": "genuinely two-answer question; either is a correct first hit" + }, + { + "id": "q32", + "question": "how do I run a backfill safely?", + "expect": ["runbooks/backfill"], + "probes": "distinctive term, but run appears everywhere" + }, + { + "id": "q33", + "question": "can I reuse the checkpoint when reprocessing?", + "expect": ["runbooks/backfill"], + "probes": "checkpoint appears in three documents; reprocessing only in the target" + }, + { + "id": "q34", + "question": "how do I run the pipeline on my laptop?", + "expect": ["notes/local-dev"], + "probes": "laptop appears once; pipeline appears in six documents" + }, + { + "id": "q35", + "question": "kafka advertised listener localhost problem", + "expect": ["notes/local-dev"], + "probes": "keyword query where the strongest term is in a low-priority personal note" + }, + { + "id": "q36", + "question": "how do I bucket timestamps in ClickHouse?", + "expect": ["notes/query-cheatsheet"], + "probes": "timestamps vs timestamp; bucket vs bucketing/buckets" + }, + { + "id": "q37", + "question": "approximate distinct count in ClickHouse", + "expect": ["notes/query-cheatsheet"], + "probes": "the document says approximate and uniq, never distinct count" + }, + { + "id": "q38", + "question": "why is my ClickHouse query slow?", + "expect": ["notes/query-cheatsheet"], + "probes": "query and slow both appear in the target; ClickHouse also appears in two team docs" + } + ] +} diff --git a/packages/core/eval/run.mjs b/packages/core/eval/run.mjs new file mode 100644 index 0000000..ac5dcd3 --- /dev/null +++ b/packages/core/eval/run.mjs @@ -0,0 +1,206 @@ +#!/usr/bin/env node + +// Retrieval eval. Scores search.mjs against a golden question set over a +// committed three-layer corpus, and checks that resolving the answer still +// surfaces the layers that disagree. +// +// Every other suite in this repo tests mechanism: does the merge pick the right +// section, does the walk stay bounded. None of them can answer "would an agent +// find the right concept from a question a person actually typed". This does. +// +// node packages/core/eval/run.mjs # report + compare to baseline.json +// node packages/core/eval/run.mjs --record --label "..." # rewrite baseline.json +// node packages/core/eval/run.mjs --json # machine-readable +// node packages/core/eval/run.mjs --verbose # show what each miss returned instead +// +// Exits nonzero when a metric falls below the recorded baseline: retrieval +// quality is a thing you can regress silently, so it is a build gate. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildSources } from "../src/sources/index.mjs"; +import { resolveConcept } from "../src/resolver.mjs"; +import { searchConcepts } from "../src/search.mjs"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.join(HERE, "baseline.json"); +const CUTOFF = 5; // recall@5: an agent realistically reads the top few hits + +// A metric may drift down by this much without failing. Ranking has ties, and a +// 38-question set moves in ~2.6% steps; anything smaller is noise, anything +// larger is a real question flipping from hit to miss. +const TOLERANCE = 0.001; + +const argv = process.argv.slice(2); +const args = new Set(argv); +const label = argv[argv.indexOf("--label") + 1]; + +const { questions } = readJson(path.join(HERE, "questions.json")); +const manifest = readJson(path.join(HERE, "manifest.json")); +const layers = buildSources(manifest, HERE); + +try { + const results = []; + for (const question of questions) { + const hits = await searchConcepts(layers, { query: question.question, limit: CUTOFF }); + const rank = hits.findIndex((hit) => question.expect.includes(hit.id)) + 1; // 0 = miss + results.push({ + ...question, + rank, + hits: hits.map((hit) => hit.id), + conflictOutcome: question.conflict ? await checkConflict(question.conflict) : null, + }); + } + + const metrics = summarize(results); + + if (args.has("--json")) { + console.log(JSON.stringify({ metrics, results }, null, 2)); + } else { + report(results, metrics); + } + + if (args.has("--record")) { + if (!label || label === "--record") { + console.error("--record needs --label \"\" so the history says why the number moved."); + process.exit(1); + } + record(metrics, results.length, label); + console.log(`\nBaseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`); + process.exit(0); + } + + process.exit(compareToBaseline(metrics) ? 0 : 1); +} finally { + for (const layer of layers) layer.close(); +} + +// ---- scoring --------------------------------------------------------------- + +// The claim ContextCake makes over a flat wiki is that disagreement survives +// retrieval. Finding the concept is only half of it: the resolved concept has to +// still carry the dissenting layers, or the answer is just the top layer's +// opinion wearing a provenance badge. +async function checkConflict({ concept, layers: expectedLayers }) { + const resolved = await resolveConcept(concept, layers); + if (!resolved) return { pass: false, reason: `concept not found: ${concept}` }; + + const seen = new Set(); + for (const section of resolved.sections) { + for (const conflict of section.conflicts ?? []) seen.add(conflict.layer); + } + const missing = expectedLayers.filter((layer) => !seen.has(layer)); + return missing.length === 0 + ? { pass: true, surfaced: [...seen] } + : { pass: false, reason: `no dissent surfaced from: ${missing.join(", ")}`, surfaced: [...seen] }; +} + +function summarize(results) { + const total = results.length; + const hitsAt = (n) => results.filter((r) => r.rank > 0 && r.rank <= n).length / total; + const mrr = results.reduce((sum, r) => sum + (r.rank > 0 ? 1 / r.rank : 0), 0) / total; + + const conflictChecks = results.filter((r) => r.conflictOutcome); + const conflictRate = conflictChecks.length === 0 + ? 1 + : conflictChecks.filter((r) => r.conflictOutcome.pass).length / conflictChecks.length; + + return { + "recall@1": round(hitsAt(1)), + [`recall@${CUTOFF}`]: round(hitsAt(CUTOFF)), + mrr: round(mrr), + conflict: round(conflictRate), + }; +} + +function round(value) { + return Math.round(value * 1000) / 1000; +} + +// ---- output ---------------------------------------------------------------- + +function report(results, metrics) { + const verbose = args.has("--verbose"); + + console.log(`\nRetrieval eval — ${results.length} questions over ${layers.length} layers\n`); + + const misses = results.filter((r) => r.rank === 0); + const demoted = results.filter((r) => r.rank > 1); + const conflictFails = results.filter((r) => r.conflictOutcome && !r.conflictOutcome.pass); + + if (misses.length) { + console.log(`Not found in the top ${CUTOFF} (${misses.length}):`); + for (const miss of misses) { + console.log(` ${miss.id} "${miss.question}"`); + console.log(` want ${miss.expect.join(" | ")}`); + console.log(` got ${miss.hits.length ? miss.hits.join(", ") : "(nothing)"}`); + if (verbose) console.log(` probes: ${miss.probes}`); + } + console.log(""); + } + + if (demoted.length) { + console.log(`Found, but not first (${demoted.length}):`); + for (const hit of demoted) { + console.log(` ${hit.id} rank ${hit.rank} "${hit.question}" → beaten by ${hit.hits[0]}`); + } + console.log(""); + } + + if (conflictFails.length) { + console.log(`Conflicts not surfaced (${conflictFails.length}):`); + for (const fail of conflictFails) { + console.log(` ${fail.id} ${fail.conflict.concept}: ${fail.conflictOutcome.reason}`); + } + console.log(""); + } + + for (const [name, value] of Object.entries(metrics)) { + console.log(` ${name.padEnd(10)} ${value.toFixed(3)}`); + } +} + +// The superseded numbers stay in the file. A retrieval change that trades one +// metric for another is a judgement call, and the next person deserves to see +// what the trade actually was rather than one row of current state. +function record(metrics, questionCount, why) { + const previous = fs.existsSync(BASELINE_PATH) ? readJson(BASELINE_PATH) : null; + const history = previous ? [{ label: previous.label, ...stripMeta(previous) }, ...(previous.history ?? [])] : []; + const next = { label: why, questions: questionCount, ...metrics, history }; + fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(next, null, 2)}\n`); +} + +function stripMeta({ label, history, ...rest }) { + return rest; +} + +function compareToBaseline(metrics) { + if (!fs.existsSync(BASELINE_PATH)) { + console.log("\nNo baseline recorded. Run with --record to set one."); + return true; + } + + const baseline = readJson(BASELINE_PATH); + const regressions = []; + const improvements = []; + for (const [name, value] of Object.entries(metrics)) { + const before = baseline[name]; + if (typeof before !== "number") continue; + if (value < before - TOLERANCE) regressions.push(`${name}: ${before.toFixed(3)} → ${value.toFixed(3)}`); + else if (value > before + TOLERANCE) improvements.push(`${name}: ${before.toFixed(3)} → ${value.toFixed(3)}`); + } + + if (improvements.length) console.log(`\nImproved: ${improvements.join(", ")}`); + if (regressions.length) { + console.log(`\nRETRIEVAL REGRESSED: ${regressions.join(", ")}`); + console.log("If the change is intentional, re-record with --record and say why in the commit."); + return false; + } + console.log("\neval ok (no regression against baseline)"); + return true; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} From 5f523c9d03ec404320c12b4fc804f2c6d46d5db6 Mon Sep 17 00:00:00 2001 From: John Siracusa Date: Wed, 29 Jul 2026 23:36:03 -0400 Subject: [PATCH 3/4] feat(core): rank with BM25F over Porter-stemmed tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces substring occurrence counting with BM25F, and the ad-hoc query tokenizer with a Porter stemmer. recall@1 0.263 -> 0.895 recall@5 0.500 -> 1.000 mrr 0.348 -> 0.947 conflict 1.000 -> 1.000 Measured, not asserted: `npm run eval`. Three changes, each addressing one cause the eval isolated: - Match tokens, not substrings. This is the largest single effect. The old scorer looked for the query word "I" with indexOf and found the "i" in identifier, in, and with, so the longest document won almost every natural-language question. - Weight by inverse document frequency, using the log(1 + ...) form so a term common across the corpus decays toward zero instead of going negative and actively demoting the documents that contain it. - Normalize by field length, b=0.75 on the body and lower on the short identifying fields, whose length carries no signal. Aggregation across layers changes from sum to best-layer. Summing let a concept that three layers happen to mention outrank the single document that answers the question; the cascade's premise is that those three are one concept, so they should not vote three times. Measured separately: BM25 alone reaches 0.842 recall@1, best-layer aggregation takes it to 0.895. The stemmer is the published Porter algorithm rather than a suffix list written alongside this corpus, and the unit test pins it against Porter's own reference vocabulary. That costs some accuracy — "rebalancing" and "rebalance" genuinely do not meet under Porter — and the test records those gaps instead of patching them. A stemmer hand-fitted to the questions grading it would make the eval measure itself. Tool schemas are untouched: fixtures/mcp-tools-baseline.json still matches byte for byte. Signed-off-by: John Siracusa Co-authored-by: Claude Opus 5 --- CLAUDE.md | 20 ++- package.json | 2 +- packages/core/eval/README.md | 75 ++++++++++ packages/core/eval/baseline.json | 19 ++- packages/core/src/search.mjs | 157 +++++++++++++++----- packages/core/src/stem.mjs | 151 +++++++++++++++++++ packages/core/tests/search.test.mjs | 220 ++++++++++++++++++++++++++++ 7 files changed, 600 insertions(+), 44 deletions(-) create mode 100644 packages/core/eval/README.md create mode 100644 packages/core/src/stem.mjs create mode 100644 packages/core/tests/search.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 36eb393..990c849 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,10 +3,19 @@ ## Commands ```bash -# Run all tests +# Run all tests (the chain lives in package.json "test" — read it there rather +# than mirroring it here, which is how the two drifted apart before) npm test -# or directly (mirrors the npm test chain): -bash packages/core/tests/smoke-test.sh && bash packages/core/tests/resolver-test.sh && bash packages/core/tests/source-test.sh && bash packages/core/tests/files-source-test.sh && bash packages/core/tests/github-source-test.sh && bash packages/core/tests/pack-test.sh && bash packages/core/tests/git-sync-test.sh && bash packages/core/tests/capture-test.sh && bash packages/core/tests/team-sync-mcp-test.sh && bash packages/core/tests/playground-test.sh && bash packages/core/tests/service-test.sh && bash packages/core/tests/mcp-respawn-test.sh && bash packages/core/tests/setup-robustness-test.sh + +# Run one suite while iterating +bash packages/core/tests/resolver-test.sh +node --test packages/core/tests/search.test.mjs + +# Retrieval eval — scores search.mjs against the golden question set. +# Part of npm test: a ranking change that loses recall fails the build. +npm run eval +node packages/core/eval/run.mjs --verbose # show what each miss returned instead +node packages/core/eval/run.mjs --record --label "why" # accept a new number, with its reason # Run the MCP server (cascade mode) node mcp-server.mjs --manifest layers.json @@ -61,6 +70,7 @@ See `docs/architecture/README.md` for the full design. Short version: - **Storage is federated** — each layer is a `source` behind a uniform adapter: an `okf-local` bundle, a plain `files` folder, a remote `github` repository, or an `mcp` foreign graph translated into OKF at read time. - **Reading is unified** — `resolver.mjs` stitches the sources into one effective OKF concept at read time. +- **Finding is ranked** — `search.mjs` (BM25F over Porter-stemmed tokens) decides which concept an agent gets to resolve in the first place. Everything the cascade is good at is downstream of it. - **Layer precedence** — Personal (3) > Team (2) > Company (0). Higher wins per section. Levels are configurable per layer in the manifest. - **Section/field merge** — not whole-document replacement. A higher layer speaks to what it knows; the rest is inherited. Where layers disagree, the primary value carries per-section `conflicts[]` (dissenting layer + date) — surfaced, not hidden. - **MCP server** — `mcp-server.mjs` exposes the resolved graph to AI agents (search, read_file, list_concepts, get_links). @@ -71,6 +81,9 @@ Key files: | File | Role | |------|------| | `packages/core/src/resolver.mjs` | Core cascade engine: section merge, precedence, provenance, conflict surfacing | +| `packages/core/src/search.mjs` | Ranking behind the `search` and `find_captures` tools: BM25F over stemmed tokens, per-field boosts and length normalization, 7-day recency half-life for captures. Importable and side-effect-free precisely so the eval can score it | +| `packages/core/src/stem.mjs` | Porter stemmer (1980). Deliberately the published algorithm, not a hand-tuned suffix list — see the note in the file | +| `packages/core/eval/` | Retrieval eval: committed 3-layer corpus, golden question set, runner, and `baseline.json` with the score history | | `packages/core/src/service.mjs` | Embeddable HTTP service: read API, background index, sources CRUD, settings, file APIs, console mount | | `packages/core/src/settings.mjs` | User-configurable engine limits (manifest > env > default) + validation and the UI catalog | | `packages/core/src/layer-files.mjs` | Layer file explorer/editor APIs (`/api/files`, `/api/file`, `/api/section`), sandboxed to layer roots | @@ -121,4 +134,5 @@ Key files: - **Aggregate reads come from a background index and are often partial.** `/api/graph` and `/api/resolve-all` answer from whatever is indexed right now and report `indexing` / `indexingSources`; clients render what they have and poll. Any assertion about completeness (tests, scripts) must pass `?wait=`. `/api/resolve` deliberately reads one concept live. An index entry is keyed by its layer config + settings, so adding a source never re-indexes the existing ones. - **The indexing limits are user settings, not env-only** (`settings.mjs`, `GET`/`PATCH /api/settings`, Settings → Indexing in the console). Precedence is manifest > env > default — the manifest has to win or the settings UI would silently do nothing. Env vars (`CONTEXTCAKE_MAX_DOC_FILES`, `CONTEXTCAKE_MAX_SCAN_ENTRIES`, `CONTEXTCAKE_SOURCE_BUDGET_MS`) remain the headless/CI fallback. - **Add-source validation is deliberately cheap**: only "folder is missing" and "that's a file" fail the form. A too-big folder is a normal thing to add — it becomes a visible source error after indexing, with a pointer at Settings. Don't reintroduce a full walk on the add path. MCP sources still probe (`tools/list`) at add time, which is bounded and catches a wrong command. +- **Retrieval is measured, not asserted.** `npm test` ends with the eval; a ranking change that loses recall fails the build with `RETRIEVAL REGRESSED`. If a change is a deliberate trade, re-record with `--record --label ""` — the superseded numbers stay in `baseline.json`'s history so the trade is visible later. Do not tune the stemmer or the field boosts against the golden questions: the set is small enough to overfit in an afternoon, and a scorer fitted to its own eval measures nothing. Add questions first, then tune. - **Layer file APIs live in the engine, not the playground** (`layer-files.mjs`), so the desktop app can browse and edit context files. They cover `files`-kind layers too — the playground's old copy only mapped `okf-local` roots, which made markdown folders invisible in the editor. diff --git a/package.json b/package.json index 1ffecad..1998e7c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Federated team knowledge with cascading layer precedence — OKF-compatible, MCP-ready.", "type": "module", "scripts": { - "test": "bash packages/core/tests/smoke-test.sh && bash packages/core/tests/resolver-test.sh && bash packages/core/tests/source-test.sh && bash packages/core/tests/files-source-test.sh && bash packages/core/tests/github-source-test.sh && node --test packages/core/tests/manifest.test.mjs && bash packages/core/tests/profile-runtime-test.sh && bash packages/core/tests/pack-test.sh && bash packages/core/tests/git-sync-test.sh && bash packages/core/tests/capture-test.sh && bash packages/core/tests/team-sync-mcp-test.sh && bash packages/core/tests/playground-test.sh && bash packages/core/tests/service-test.sh && bash packages/core/tests/mcp-respawn-test.sh && bash packages/core/tests/setup-robustness-test.sh && npm run eval", + "test": "bash packages/core/tests/smoke-test.sh && bash packages/core/tests/resolver-test.sh && bash packages/core/tests/source-test.sh && bash packages/core/tests/files-source-test.sh && bash packages/core/tests/github-source-test.sh && node --test packages/core/tests/manifest.test.mjs && node --test packages/core/tests/search.test.mjs && bash packages/core/tests/profile-runtime-test.sh && bash packages/core/tests/pack-test.sh && bash packages/core/tests/git-sync-test.sh && bash packages/core/tests/capture-test.sh && bash packages/core/tests/team-sync-mcp-test.sh && bash packages/core/tests/playground-test.sh && bash packages/core/tests/service-test.sh && bash packages/core/tests/mcp-respawn-test.sh && bash packages/core/tests/setup-robustness-test.sh && npm run eval", "eval": "node packages/core/eval/run.mjs", "mcp": "node mcp-server.mjs", "playground": "node apps/playground/server.mjs", diff --git a/packages/core/eval/README.md b/packages/core/eval/README.md new file mode 100644 index 0000000..a741ad2 --- /dev/null +++ b/packages/core/eval/README.md @@ -0,0 +1,75 @@ +# Retrieval eval + +Every other suite in this repo tests mechanism: does the merge pick the right +section, does the walk stay bounded, does the token guard hold. None of them can +answer the question the product actually lives or dies on — **would an agent find +the right concept from a question a person actually typed?** + +This measures that. + +```bash +npm run eval # report + regression gate +node packages/core/eval/run.mjs --verbose # show what each miss returned instead +node packages/core/eval/run.mjs --json # machine-readable +node packages/core/eval/run.mjs --record --label "why" # accept a new number +``` + +## What is here + +| Path | Role | +|------|------| +| `corpus/` | Three committed OKF layers — company (0), team (2), personal (3) — with deliberate cross-layer overrides | +| `questions.json` | 38 natural-language questions, each with the concept ids that answer it and a `probes` note saying why the question exists | +| `manifest.json` | Points the engine at the corpus; also usable by hand with `resolver.mjs` or `mcp-server.mjs` | +| `run.mjs` | Runner and regression gate | +| `baseline.json` | Current metrics plus the history of every superseded number | + +## Metrics + +- **recall@1** — the right concept is the first hit. +- **recall@5** — it is in the top five, which is realistically what an agent reads. +- **mrr** — mean reciprocal rank; moves when a hit is merely demoted rather than lost. +- **conflict** — for questions where layers disagree, resolving the answer still + surfaces the dissenting layers. Finding the concept is only half the claim: if + the disagreement does not survive retrieval, the answer is just the top layer's + opinion wearing a provenance badge. + +## Results so far + +| Scorer | recall@1 | recall@5 | mrr | conflict | +|---|---|---|---|---| +| Substring occurrence counting | 0.263 | 0.500 | 0.348 | 1.000 | +| BM25F over Porter-stemmed tokens | 0.895 | 1.000 | 0.947 | 1.000 | + +The old scorer counted `indexOf` hits with fixed field weights. Three things +were wrong with it, and the eval separated them: + +1. **Substring matching, not token matching.** The query word `I` matched the + `i` inside *identifier*, *in*, and *with*. Natural questions carry several + such words, so the longest document usually won. +2. **No inverse document frequency.** A word in every document counted as much + as a word in one. +3. **No length normalization.** Repetition beat precision. + +`conflict` was already 1.000 before the rewrite. That is the honest read on the +engine: the cascade and the conflict surfacing worked; retrieval was the broken +half, and it was broken badly enough to hide the working half. + +## Adding questions + +Write the question the way someone would actually type it, *then* see what the +ranker does. A question written after looking at the corpus tends to reuse the +document's own vocabulary, which grades the ranker on the one case it is +guaranteed to pass. + +Fill in `probes` with what the question is testing — a morphological variant, a +distractor term, a genuine synonym gap. When a question flips from hit to miss, +that note is what tells you whether the change was real. + +## Known ceilings + +Some questions cannot be won by lexical ranking at any amount of tuning. `q07` +("how long do we keep logs?") asks with *keep* against a corpus that says +*retained*; no stemmer bridges that. Those questions stay in the set on purpose +— they are the honest measure of what a lexical ranker leaves on the table, and +the number to beat if embeddings ever earn their dependency. diff --git a/packages/core/eval/baseline.json b/packages/core/eval/baseline.json index bf3f913..ab4bfb1 100644 --- a/packages/core/eval/baseline.json +++ b/packages/core/eval/baseline.json @@ -1,9 +1,18 @@ { - "label": "substring occurrence counting (pre-BM25)", + "label": "BM25F over Porter-stemmed tokens; best-layer aggregation", "questions": 38, - "recall@1": 0.263, - "recall@5": 0.5, - "mrr": 0.348, + "recall@1": 0.895, + "recall@5": 1, + "mrr": 0.947, "conflict": 1, - "history": [] + "history": [ + { + "label": "substring occurrence counting (pre-BM25)", + "questions": 38, + "recall@1": 0.263, + "recall@5": 0.5, + "mrr": 0.348, + "conflict": 1 + } + ] } diff --git a/packages/core/src/search.mjs b/packages/core/src/search.mjs index 1236f29..f22dd64 100644 --- a/packages/core/src/search.mjs +++ b/packages/core/src/search.mjs @@ -3,19 +3,48 @@ // // This lives outside mcp-server.mjs on purpose. That module parses argv and can // call process.exit() at load time, so it cannot be imported — which meant the -// ranking had no way to be measured. +// ranking had no way to be measured. The eval harness in packages/core/eval/ +// scores this module directly against a golden question set; change the ranking +// and re-run it before believing the change helped. // // Not to be confused with tokenize.mjs, which is the BPE tokenizer used for -// context-budget accounting. The tokenizer here is a query analyzer. +// context-budget accounting. The analyzer here is a query analyzer. -const DAY_MS = 86400000; - -// Field order is load-bearing: scorers weight by position. 0-2 are the -// identifying fields (id, title, description), 3 is tags, 4 is the body. -const WEIGHTS = [4, 4, 4, 2, 1]; +import { stem } from "./stem.mjs"; +const DAY_MS = 86400000; +const WORD = /[a-z0-9_-]+/g; + +// BM25F. Per-field boosts say where a match counts for more; per-field `b` says +// how hard to punish length. Body gets the standard 0.75 because a long +// document should not outrank a precise one merely by repeating a word; the +// short identifying fields get less, since their length carries no signal. +const FIELDS = [ + { key: "id", boost: 3, b: 0.4 }, + { key: "title", boost: 5, b: 0.4 }, + { key: "description", boost: 3, b: 0.5 }, + { key: "tags", boost: 2, b: 0.4 }, + { key: "body", boost: 1, b: 0.75 }, +]; +const K1 = 1.2; + +// Raw query words, kept unstemmed for snippet highlighting — a snippet has to +// point at text the reader can actually see. export function tokenizeQuery(query) { - return query.toLowerCase().match(/[a-z0-9_-]+/g) ?? []; + return query.toLowerCase().match(WORD) ?? []; +} + +// Index terms. Hyphenated compounds also contribute their parts so that +// "exactly-once" is reachable from "exactly once" and the reverse. +export function analyze(text) { + const terms = []; + for (const token of String(text).toLowerCase().match(WORD) ?? []) { + terms.push(stem(token)); + if (token.includes("-")) { + for (const part of token.split("-")) if (part) terms.push(stem(part)); + } + } + return terms; } function requireTokens(query, tool) { @@ -25,8 +54,8 @@ function requireTokens(query, tool) { return tokens; } -// One I/O pass over the cascade. Callers score the returned array in memory, so -// adding a second scoring pass costs nothing on top of the walk. +// One I/O pass over the cascade. Indexing and scoring happen in memory over the +// returned array, so the extra passes cost nothing on top of the walk. async function collectDocuments(layers, { prefix = null } = {}) { const docs = []; for (const source of layers) { @@ -58,27 +87,70 @@ function conceptFields(doc) { function captureFields(doc) { // Captures carry no description or tags; keeping the arity fixed keeps the - // positional weights aligned with concept scoring. + // positional boosts aligned with concept scoring. return [doc.id, doc.frontmatter.title ?? "", "", "", doc.body]; } -export function scoreFields(tokens, fields) { - return tokens.reduce((total, token) => { - return total + fields.reduce((subtotal, field, index) => { - return subtotal + countOccurrences(String(field).toLowerCase(), token) * WEIGHTS[index]; - }, 0); - }, 0); +// ---- BM25F ----------------------------------------------------------------- + +function buildIndex(docs, fieldsOf) { + const fieldTotals = FIELDS.map(() => 0); + const entries = docs.map((doc) => { + const fields = fieldsOf(doc).map((value, index) => { + const terms = analyze(value); + fieldTotals[index] += terms.length; + const frequencies = new Map(); + for (const term of terms) frequencies.set(term, (frequencies.get(term) ?? 0) + 1); + return { frequencies, length: terms.length }; + }); + return { doc, fields }; + }); + + const documentFrequency = new Map(); + for (const entry of entries) { + const seen = new Set(); + for (const field of entry.fields) for (const term of field.frequencies.keys()) seen.add(term); + for (const term of seen) documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1); + } + + return { + total: entries.length, + averageLength: fieldTotals.map((sum) => (entries.length ? sum / entries.length : 0)), + documentFrequency, + entries, + }; +} + +// This IDF form stays positive even for a term that appears in most documents, +// which matters here: the alternative goes negative past df > N/2 and lets a +// common word actively push a document down the list. +function inverseDocumentFrequency(index, term) { + const df = index.documentFrequency.get(term) ?? 0; + if (df === 0) return 0; + return Math.log(1 + (index.total - df + 0.5) / (df + 0.5)); } -function countOccurrences(haystack, needle) { - if (!needle) return 0; - let count = 0; - let index = haystack.indexOf(needle); - while (index !== -1) { - count += 1; - index = haystack.indexOf(needle, index + needle.length); +export function scoreEntry(index, entry, terms) { + let score = 0; + for (const term of terms) { + const idf = inverseDocumentFrequency(index, term); + if (idf === 0) continue; + + // Accumulate length-normalized frequency across fields first, then saturate + // once. Saturating per field would let a term in five fields outscore the + // same term used meaningfully in one. + let weighted = 0; + for (let position = 0; position < FIELDS.length; position += 1) { + const field = entry.fields[position]; + const frequency = field.frequencies.get(term); + if (!frequency) continue; + const { boost, b } = FIELDS[position]; + const average = index.averageLength[position] || 1; + weighted += (boost * frequency) / (1 - b + b * (field.length / average)); + } + if (weighted > 0) score += (idf * weighted) / (K1 + weighted); } - return count; + return score; } export function makeSnippet(body, tokens) { @@ -96,14 +168,18 @@ function layerOrderer(layers) { } export async function searchConcepts(layers, { query, limit = 10 }) { - const tokens = requireTokens(query, "search"); + const rawTokens = requireTokens(query, "search"); + const terms = [...new Set(analyze(query))]; const orderLayerNames = layerOrderer(layers); + const docs = await collectDocuments(layers); + const index = buildIndex(docs, conceptFields); const byId = new Map(); - for (const doc of docs) { - const score = scoreFields(tokens, conceptFields(doc)); + for (const entry of index.entries) { + const score = scoreEntry(index, entry, terms); if (score <= 0) continue; + const { doc } = entry; const existing = byId.get(doc.id); if (!existing) { @@ -112,10 +188,17 @@ export async function searchConcepts(layers, { query, limit = 10 }) { title: doc.frontmatter.title ?? null, score, layers: [doc.layer], - snippet: makeSnippet(doc.body, tokens), + snippet: makeSnippet(doc.body, rawTokens), }); } else { - existing.score += score; + // Best layer wins rather than the sum. Summing made a concept that three + // layers happen to mention outrank the one document that answers the + // question — the cascade's whole point is that those three are one + // concept, so they should not vote three times. + if (score > existing.score) { + existing.score = score; + existing.snippet = makeSnippet(doc.body, rawTokens); + } existing.layers.push(doc.layer); if (!existing.title) existing.title = doc.frontmatter.title ?? null; } @@ -128,14 +211,18 @@ export async function searchConcepts(layers, { query, limit = 10 }) { } export async function searchCaptures(layers, { query, kinds = null, limit = 10, now = Date.now() }) { - const tokens = requireTokens(query, "find_captures"); + const rawTokens = requireTokens(query, "find_captures"); + const terms = [...new Set(analyze(query))]; + const docs = await collectDocuments(layers, { prefix: "captures/" }); + const eligible = kinds ? docs.filter((doc) => kinds.includes(doc.frontmatter.kind)) : docs; + const index = buildIndex(eligible, captureFields); const rows = []; - for (const doc of docs) { - if (kinds && !kinds.includes(doc.frontmatter.kind)) continue; - const base = scoreFields(tokens, captureFields(doc)); + for (const entry of index.entries) { + const base = scoreEntry(index, entry, terms); if (base <= 0) continue; + const { doc } = entry; const capturedAt = doc.frontmatter.captured ?? null; const capturedTime = capturedAt ? new Date(capturedAt).getTime() : NaN; @@ -151,7 +238,7 @@ export async function searchCaptures(layers, { query, kinds = null, limit = 10, ageDays: Math.round(ageDays * 10) / 10, status: doc.frontmatter.status ?? "unreviewed", score: base * 2 ** (-ageDays / 7), // true 7-day half-life - snippet: makeSnippet(doc.body, tokens), + snippet: makeSnippet(doc.body, rawTokens), layer: doc.layer, }); } diff --git a/packages/core/src/stem.mjs b/packages/core/src/stem.mjs new file mode 100644 index 0000000..44f2cb8 --- /dev/null +++ b/packages/core/src/stem.mjs @@ -0,0 +1,151 @@ +// Porter stemmer (Porter, 1980), so a query for "databases" reaches a document +// that says "database" and "rebalancing" reaches "rebalance". +// +// Deliberately the published algorithm rather than a hand-tuned suffix list: +// the retrieval eval is scored on questions written by hand, and a stemmer +// invented alongside those questions would be measuring itself. Porter predates +// this corpus by forty-odd years and cannot have been fitted to it. +// +// Dependency-free, like the rest of the engine. + +const VOWEL = "aeiou"; + +function isConsonant(word, index) { + const letter = word[index]; + if (VOWEL.includes(letter)) return false; + // 'y' is a consonant unless the letter before it is one. + if (letter !== "y") return true; + return index === 0 ? true : !isConsonant(word, index - 1); +} + +// The "measure": how many vowel-consonant sequences the stem contains. Porter +// uses it to avoid stripping suffixes off words that are too short to survive it. +function measure(word) { + let count = 0; + let index = 0; + const length = word.length; + + while (index < length && isConsonant(word, index)) index += 1; + while (index < length) { + while (index < length && !isConsonant(word, index)) index += 1; + if (index >= length) break; + count += 1; + while (index < length && isConsonant(word, index)) index += 1; + } + return count; +} + +function hasVowel(word) { + for (let index = 0; index < word.length; index += 1) { + if (!isConsonant(word, index)) return true; + } + return false; +} + +function endsWithDoubleConsonant(word) { + if (word.length < 2) return false; + const last = word.length - 1; + return word[last] === word[last - 1] && isConsonant(word, last); +} + +// consonant-vowel-consonant where the final consonant is not w, x or y. +function endsCvc(word) { + if (word.length < 3) return false; + const last = word.length - 1; + if (!isConsonant(word, last) || isConsonant(word, last - 1) || !isConsonant(word, last - 2)) return false; + return !"wxy".includes(word[last]); +} + +function replaceSuffix(word, suffix, replacement, minMeasure) { + if (!word.endsWith(suffix)) return null; + const stem = word.slice(0, word.length - suffix.length); + if (minMeasure !== undefined && measure(stem) <= minMeasure) return null; + return stem + replacement; +} + +const STEP2 = [ + ["ational", "ate"], ["tional", "tion"], ["enci", "ence"], ["anci", "ance"], + ["izer", "ize"], ["bli", "ble"], ["alli", "al"], ["entli", "ent"], + ["eli", "e"], ["ousli", "ous"], ["ization", "ize"], ["ation", "ate"], + ["ator", "ate"], ["alism", "al"], ["iveness", "ive"], ["fulness", "ful"], + ["ousness", "ous"], ["aliti", "al"], ["iviti", "ive"], ["biliti", "ble"], + ["logi", "log"], +]; + +const STEP3 = [ + ["icate", "ic"], ["ative", ""], ["alize", "al"], ["iciti", "ic"], + ["ical", "ic"], ["ful", ""], ["ness", ""], +]; + +const STEP4 = [ + "al", "ance", "ence", "er", "ic", "able", "ible", "ant", "ement", + "ment", "ent", "ou", "ism", "ate", "iti", "ous", "ive", "ize", +]; + +export function stem(input) { + let word = input; + // Too short to have a suffix worth removing, and acronyms like "tls" must not + // lose their trailing s. + if (word.length <= 3) return word; + + // Step 1a — plurals. + word = replaceSuffix(word, "sses", "ss") ?? replaceSuffix(word, "ies", "i") + ?? (word.endsWith("ss") ? word : replaceSuffix(word, "s", "")) ?? word; + + // Step 1b — past tense and gerunds. + let step1bApplied = false; + const eed = replaceSuffix(word, "eed", "ee", 0); + if (eed !== null) { + word = eed; + } else { + for (const suffix of ["ed", "ing"]) { + if (!word.endsWith(suffix)) continue; + const stripped = word.slice(0, word.length - suffix.length); + if (!hasVowel(stripped)) continue; + word = stripped; + step1bApplied = true; + break; + } + } + if (step1bApplied) { + if (word.endsWith("at") || word.endsWith("bl") || word.endsWith("iz")) word += "e"; + else if (endsWithDoubleConsonant(word) && !"lsz".includes(word[word.length - 1])) word = word.slice(0, -1); + else if (measure(word) === 1 && endsCvc(word)) word += "e"; + } + + // Step 1c — terminal y to i. + if (word.endsWith("y") && hasVowel(word.slice(0, -1))) word = `${word.slice(0, -1)}i`; + + // Step 2 and 3 — derivational suffixes, longest match first. + for (const [suffix, replacement] of STEP2) { + const next = replaceSuffix(word, suffix, replacement, 0); + if (next !== null) { word = next; break; } + } + for (const [suffix, replacement] of STEP3) { + const next = replaceSuffix(word, suffix, replacement, 0); + if (next !== null) { word = next; break; } + } + + // Step 4 — strip the suffix entirely when the stem is substantial enough. + for (const suffix of STEP4) { + if (!word.endsWith(suffix)) continue; + const stripped = word.slice(0, word.length - suffix.length); + if (measure(stripped) <= 1) continue; + word = stripped; + break; + } + if (word.endsWith("ion")) { + const stripped = word.slice(0, -3); + if (measure(stripped) > 1 && (stripped.endsWith("s") || stripped.endsWith("t"))) word = stripped; + } + + // Step 5 — tidy up a trailing e and a doubled l. + if (word.endsWith("e")) { + const stripped = word.slice(0, -1); + const m = measure(stripped); + if (m > 1 || (m === 1 && !endsCvc(stripped))) word = stripped; + } + if (measure(word) > 1 && endsWithDoubleConsonant(word) && word.endsWith("l")) word = word.slice(0, -1); + + return word; +} diff --git a/packages/core/tests/search.test.mjs b/packages/core/tests/search.test.mjs new file mode 100644 index 0000000..919a49d --- /dev/null +++ b/packages/core/tests/search.test.mjs @@ -0,0 +1,220 @@ +// Unit tests for the retrieval module. The eval in packages/core/eval/ measures +// whether ranking is *good*; these pin the properties it must never lose, so a +// regression names itself instead of showing up as a metric that slipped. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { stem } from "../src/stem.mjs"; +import { analyze, searchConcepts, searchCaptures } from "../src/search.mjs"; + +// A slice of Porter's own published vocabulary. If these drift, the stemmer has +// stopped being Porter and the eval numbers are measuring something else. +const PORTER_REFERENCE = [ + ["caresses", "caress"], ["ponies", "poni"], ["ties", "ti"], ["cats", "cat"], + ["feed", "fe"], ["agreed", "agre"], ["plastered", "plaster"], ["motoring", "motor"], + ["sing", "sing"], ["conflated", "conflat"], ["troubled", "troubl"], ["sized", "size"], + ["hopping", "hop"], ["tanned", "tan"], ["falling", "fall"], ["hissing", "hiss"], + ["fizzed", "fizz"], ["failing", "fail"], ["filing", "file"], ["happy", "happi"], + ["sky", "sky"], ["relational", "relat"], ["conditional", "condit"], ["rational", "ration"], + ["digitizer", "digit"], ["vietnamization", "vietnam"], ["predication", "predic"], + ["operator", "oper"], ["feudalism", "feudal"], ["decisiveness", "decis"], + ["hopefulness", "hope"], ["callousness", "callous"], ["formaliti", "formal"], + ["sensitiviti", "sensit"], ["sensibiliti", "sensibl"], ["triplicate", "triplic"], + ["formative", "form"], ["formalize", "formal"], ["electriciti", "electr"], + ["electrical", "electr"], ["hopeful", "hope"], ["goodness", "good"], + ["revival", "reviv"], ["allowance", "allow"], ["inference", "infer"], + ["airliner", "airlin"], ["gyroscopic", "gyroscop"], ["adjustable", "adjust"], + ["defensible", "defens"], ["irritant", "irrit"], ["replacement", "replac"], + ["dependent", "depend"], ["adoption", "adopt"], ["communism", "commun"], + ["activate", "activ"], ["homologous", "homolog"], ["effective", "effect"], + ["bowdlerize", "bowdler"], ["probate", "probat"], ["rate", "rate"], + ["cease", "ceas"], ["controll", "control"], ["roll", "roll"], +]; + +test("the stemmer is Porter, not an invention", () => { + for (const [input, expected] of PORTER_REFERENCE) { + assert.equal(stem(input), expected, `stem(${input})`); + } +}); + +test("short tokens and acronyms keep their final s", () => { + // "tls" must not become "tl": three-letter acronyms are common in this corpus + // and stripping the s makes them collide with unrelated words. + assert.equal(stem("tls"), "tls"); + assert.equal(stem("api"), "api"); + assert.equal(stem("aws"), "aws"); +}); + +test("a query reaches a document written in a different inflection", () => { + const pairs = [ + ["databases", "database"], + ["skewed", "skew"], + ["paginate", "pagination"], + ["compatible", "compatibility"], + ["timestamps", "timestamp"], + ["reviews", "review"], + ["deployments", "deployment"], + ["reprocessing", "reprocess"], + ]; + for (const [asked, written] of pairs) { + assert.deepEqual(analyze(asked), analyze(written), `${asked} vs ${written}`); + } +}); + +test("stemming is not synonymy, and the gaps are Porter's own", () => { + // Porter strips -ing at step 1b but -ance at step 4, so these two forms of the + // same word do not meet. Recorded rather than patched: the moment the stemmer + // is hand-adjusted to this repo's vocabulary, the eval starts grading a + // stemmer that was fitted to the questions it is being graded on. + assert.notDeepEqual(analyze("rebalancing"), analyze("rebalance")); + + // And no stemmer bridges a genuine synonym. "keep" will not reach "retained"; + // closing that gap needs a different mechanism, not a bigger suffix list. + assert.notDeepEqual(analyze("keep"), analyze("retained")); +}); + +test("a hyphenated compound is reachable from its parts", () => { + const compound = analyze("exactly-once"); + for (const part of analyze("exactly once")) { + assert.ok(compound.includes(part), `expected ${part} in ${compound.join(",")}`); + } +}); + +// ---- fixtures -------------------------------------------------------------- + +function layer(name, level, docs) { + return { + name, + level, + async listConceptIds() { + return Object.keys(docs); + }, + async loadConcept(id) { + const doc = docs[id]; + if (!doc) return null; + return { + frontmatter: doc.frontmatter ?? {}, + sections: [{ key: "body", heading: null, lines: (doc.body ?? "").split("\n") }], + }; + }, + close() {}, + }; +} + +const padding = "The platform team reviews this document every quarter as part of the standing operational review. "; + +test("a precise short document outranks a long one that mentions the term in passing", async () => { + const layers = [ + layer("company", 0, { + "notes/sprawl": { + frontmatter: { title: "Operational miscellany" }, + // Says "checkpoint" more times than the precise doc, but says everything + // else too. Occurrence counting ranked this first; length normalization + // is what stops it. + body: `${padding.repeat(12)} checkpoint. ${padding.repeat(12)} checkpoint. ${padding.repeat(12)} checkpoint.`, + }, + "runbooks/checkpoint": { + frontmatter: { title: "Checkpoint recovery" }, + body: "Restore the job from its last checkpoint before investigating.", + }, + }), + ]; + + const hits = await searchConcepts(layers, { query: "checkpoint", limit: 5 }); + assert.equal(hits[0].id, "runbooks/checkpoint"); +}); + +test("a word in every document does not decide the ranking", async () => { + const layers = [ + layer("company", 0, { + "a/one": { frontmatter: { title: "One" }, body: `service service service ${padding}` }, + "a/two": { frontmatter: { title: "Two" }, body: `service service ${padding}` }, + "a/three": { frontmatter: { title: "Three" }, body: `service kafka ${padding}` }, + }), + ]; + + // "service" is in all three, so it carries almost no information; "kafka" + // is in one and must decide the winner. + const hits = await searchConcepts(layers, { query: "service kafka", limit: 5 }); + assert.equal(hits[0].id, "a/three"); +}); + +test("a concept several layers speak to is returned once, highest layer first", async () => { + const docs = { "decisions/stack": { frontmatter: { title: "Stack" }, body: "kafka streaming platform" } }; + const layers = [ + layer("company", 0, docs), + layer("personal", 3, docs), + layer("team", 2, docs), + ]; + + const hits = await searchConcepts(layers, { query: "kafka", limit: 5 }); + assert.equal(hits.length, 1); + assert.deepEqual(hits[0].layers, ["personal", "team", "company"]); +}); + +test("an empty or unsearchable query is refused rather than matching everything", async () => { + const layers = [layer("company", 0, { "a/one": { body: "anything" } })]; + await assert.rejects(() => searchConcepts(layers, { query: "" }), /non-empty query/); + await assert.rejects(() => searchConcepts(layers, { query: " " }), /at least one searchable token/); + await assert.rejects(() => searchCaptures(layers, { query: "!!!" }), /at least one searchable token/); +}); + +test("captures decay: same relevance, the fresher one wins", async () => { + const now = Date.parse("2026-07-29T00:00:00Z"); + const layers = [ + layer("live", 1, { + "captures/old": { + frontmatter: { title: "Rebalance storm", kind: "gotcha", captured: "2026-06-01T00:00:00Z" }, + body: "consumer group rebalance storm after deploy", + }, + "captures/new": { + frontmatter: { title: "Rebalance storm", kind: "gotcha", captured: "2026-07-28T00:00:00Z" }, + body: "consumer group rebalance storm after deploy", + }, + }), + ]; + + const hits = await searchCaptures(layers, { query: "rebalance storm", limit: 5, now }); + assert.equal(hits[0].id, "captures/new"); + assert.ok(hits[0].score > hits[1].score * 2, "a two-month-old capture should be well below a one-day-old one"); +}); + +test("captures with an unparseable date still surface instead of poisoning the sort", async () => { + const now = Date.parse("2026-07-29T00:00:00Z"); + const layers = [ + layer("live", 1, { + "captures/broken": { + frontmatter: { title: "Rebalance", kind: "gotcha", captured: "not a date" }, + body: "consumer group rebalance storm", + }, + }), + ]; + + const hits = await searchCaptures(layers, { query: "rebalance", limit: 5, now }); + assert.equal(hits.length, 1); + assert.ok(Number.isFinite(hits[0].score)); +}); + +test("the kinds filter excludes other capture kinds", async () => { + const layers = [ + layer("live", 1, { + "captures/a": { frontmatter: { title: "A", kind: "gotcha", captured: "2026-07-01T00:00:00Z" }, body: "kafka lag" }, + "captures/b": { frontmatter: { title: "B", kind: "decision", captured: "2026-07-01T00:00:00Z" }, body: "kafka lag" }, + }), + ]; + + const hits = await searchCaptures(layers, { query: "kafka lag", kinds: ["decision"], limit: 5 }); + assert.deepEqual(hits.map((hit) => hit.id), ["captures/b"]); +}); + +test("only captures/ documents reach find_captures", async () => { + const layers = [ + layer("live", 1, { + "captures/a": { frontmatter: { title: "A", kind: "gotcha", captured: "2026-07-01T00:00:00Z" }, body: "kafka lag" }, + "decisions/streaming": { frontmatter: { title: "Streaming" }, body: "kafka lag" }, + }), + ]; + + const hits = await searchCaptures(layers, { query: "kafka lag", limit: 5 }); + assert.deepEqual(hits.map((hit) => hit.id), ["captures/a"]); +}); From e670330bfff93f606b3d1e0ce6800a9faa07ce40 Mon Sep 17 00:00:00 2001 From: John Siracusa Date: Wed, 29 Jul 2026 23:37:08 -0400 Subject: [PATCH 4/4] ci: block a release while a gate is open PR #38 merged deferring three hosted acceptance checks to #44, noting they "remain required before publishing the desktop account-sync release." Three releases shipped anyway: 0.1.0, 0.2.0 and 0.3.0, all with account sync reachable by anyone who downloads them. Reachable, not merely present. generate-supabase-config.mjs throws unless SUPABASE_URL and SUPABASE_ANON_KEY are set, app-release.yml supplies both from repository secrets, and AccountPanel renders unconditionally in Settings with sign in, sign out and delete account. There is no flag; every published build can sign a user into the hosted project. Nothing here says the feature is broken. It says nobody has confirmed it works on the artifact users actually download. Records the gate in docs/release-gates.md, splitting it by what can prove what: - Callback state validation, cancel/retry races, offline sign-out, and the whole scrub/quarantine model that keeps paths and MCP commands from syncing as runnable config are already covered by tests in apps/desktop/test/ and need no manual repeat. - A packaged app, a second physical Mac, and the hosted database cannot be simulated. Those three checks stay manual, with commands. Then makes the file the gate rather than a note about one: the release workflow refuses to publish while an "## Open:" section remains. The issue-only version of this gate was enforceable solely by remembering it, which is precisely how it came to be missed three times. Signed-off-by: John Siracusa Co-authored-by: Claude Opus 5 --- .github/workflows/app-release.yml | 13 +++++ docs/release-gates.md | 92 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 docs/release-gates.md diff --git a/.github/workflows/app-release.yml b/.github/workflows/app-release.yml index 6749778..3c4bcc3 100644 --- a/.github/workflows/app-release.yml +++ b/.github/workflows/app-release.yml @@ -49,6 +49,19 @@ jobs: exit 1 fi + - name: Verify no release gate is open + run: | + # A gate declared only in a GitHub issue is enforceable only by + # someone remembering it, which is exactly how the hosted-OAuth + # checks in #44 came to be deferred through three releases. The + # gates live in docs/release-gates.md; an open one stops the build. + if grep -q '^## Open:' docs/release-gates.md; then + echo "::error::Release gate open in docs/release-gates.md:" + grep '^## Open:' docs/release-gates.md >&2 + echo "Close the gate (move it under '## Closed:' with its evidence) or, to ship anyway, say so explicitly in that file." >&2 + exit 1 + fi + - name: Setup Node uses: actions/setup-node@v7 with: diff --git a/docs/release-gates.md b/docs/release-gates.md new file mode 100644 index 0000000..e48c12e --- /dev/null +++ b/docs/release-gates.md @@ -0,0 +1,92 @@ +# Release gates + +A gate is a check that must pass before a release is published, recorded here so +the answer to "was this validated?" is a file rather than a memory. + +**This file is enforced.** `app-release.yml` refuses to publish while any +`## Open:` section remains below. Closing a gate means moving it under +`## Closed:` with its evidence — not deleting it. To ship with a gate knowingly +unmet, say so in the file; the point is that the decision is written down. + +## Open: hosted OAuth and settings sync (#44) + +**Status: breached.** PR #38 merged 2026-07-15 deferring three hosted acceptance +checks to #44 with the note that they "remain required before publishing the +desktop account-sync release." Three releases have shipped since: + +| Release | Date | Account sync reachable? | +|---|---|---| +| 0.1.0 | 2026-07-17 | yes | +| 0.2.0 | 2026-07-20 | yes | +| 0.3.0 | 2026-07-29 | yes | + +Reachable, not merely present: `apps/desktop/scripts/generate-supabase-config.mjs` +throws unless `SUPABASE_URL` and `SUPABASE_ANON_KEY` are set, `app-release.yml` +supplies both from repository secrets, and `AccountPanel` renders unconditionally +in Settings with Sign in, Sign out, and Delete account. Every published build can +sign a user into the hosted project. + +This is a process failure, not a defect report. Nothing here says the feature is +broken — it says nobody has confirmed it works on the artifact users download. + +### What CI already discharges + +These sub-checks are covered by automated tests on every PR (`apps/desktop/test/`, +run by the `desktop` job in `ci.yml`) and do not need to be repeated by hand: + +| Sub-check | Covered by | +|---|---| +| Callback `state` validation; only an encrypted session is written | `auth.test.mjs` — OAuth IPC smoke | +| Cancel and retry cannot orphan or hijack an in-flight attempt | `auth.test.mjs` — duplicate sign-in, late/stale refresh, sign-out races | +| Sign-out clears the encrypted local session, including offline | `auth.test.mjs` — sign-out with Supabase offline | +| Paths, MCP commands/args, cache dirs and credential references never sync as runnable config | `settings-sync.test.mjs` — scrub, allowlist, quarantine, path-shaped keys | +| A second client converges: tombstones, account switch, last-write-wins | `settings-sync.test.mjs` — dirty tombstone, shadow discard, remote-row precedence | + +### What still needs a human + +Three things cannot be simulated: a **packaged** app, a **second physical +machine**, and the **hosted** database. Each needs John — the OAuth flow requires +entering real credentials, which is not something an agent should do on his +behalf. + +**1. Packaged OAuth and Keychain persistence** + +```bash +cd apps/desktop && SUPABASE_URL= SUPABASE_ANON_KEY= npm run dist +``` + +Then, on the installed app: complete the GitHub browser flow, confirm +`contextcake://auth/callback` returns to the app, quit and relaunch, and confirm +the session survived. Then cancel a sign-in mid-flow and confirm nothing persists. + +**2. Second-machine settings roundtrip** + +Sign in to the same account on a second Mac or macOS user. Change a preference on +each side and confirm both directions converge. Confirm the second machine's +sources arrive as non-runnable pending metadata, not as executable configuration. + +**3. Hosted account deletion** + +With a disposable account, invoke Delete account. Confirm in the hosted project +that the Auth user and the `user_settings` row are both gone, that local session +state is cleared, and that the app still works signed out. + +### Evidence to record when closing + +Packaged version and commit SHA · macOS versions and devices used · redacted +screenshots or logs per check · confirmation the hosted records were removed. + +### Why it slipped + +Nothing connected the open issue to the release workflow. A gate declared only in +an issue is a gate only a person can enforce, and the person was busy shipping. + +Fixed by the "Verify no release gate is open" step in `app-release.yml`, added +alongside this file: tagging `app-v0.3.1` now fails while the section above still +says `## Open:`. That is the intended behavior, not an obstacle to route around — +the next release is blocked until these three checks are done or the decision to +ship without them is written here. + +## Closed: + +*(none yet — the first gate to close will be #44, above)*