diff --git a/CLAUDE.md b/CLAUDE.md index 0871443..f391a6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ Key files: |------|------| | `packages/core/src/resolver.mjs` | Core cascade engine: section merge, precedence, provenance, conflict surfacing | | `packages/core/src/mcp-server.mjs` | stdio MCP server; resolves via resolver.mjs; renders conflicts in markdown | -| `packages/core/src/sources/okf-local.mjs` | OKF-local source adapter: reads OKF markdown bundles from disk | +| `packages/core/src/sources/okf-local.mjs` | OKF-local source adapter: reads OKF markdown bundles from disk. Owns `withDocumentDate` (the section-date fallback chain, shared with every other adapter) and dates undated sections from git history (author dates, batched + TTL-memoized), never the mtime | | `packages/core/src/sources/mcp.mjs` | MCP source adapter: spawns a foreign stdio MCP server, translates to OKF | | `packages/core/src/sources/files.mjs` | Files source adapter: any plain folder of `.md`/`.mdx`/`.txt` docs becomes a layer (OKF parsing when frontmatter present, synthesized sections otherwise). Owns `parseDocument`, shared with remote adapters so section keys match | | `packages/core/src/sources/github.mjs` | GitHub source adapter: a repo's `CLAUDE.md`/`AGENTS.md`/`README.md`/`docs/**` become a layer without a clone — recursive tree index, raw content, last-commit dates, warn-and-continue on API failure | @@ -103,7 +103,7 @@ Key files: - `layers.json` contains absolute paths — gitignored, each developer has their own. - `apps/control-surface/signals.json` is generated — gitignored, produced by `ingest.mjs`. -- Staleness is surfaced via per-section `conflicts[]` + last-updated dates (the shadow/hash subsystem was removed in the core re-arch; see `specs/contextcake-core/design.md`). +- Staleness is surfaced via per-section `conflicts[]` + last-updated dates (the shadow/hash subsystem was removed in the core re-arch; see `specs/contextcake-core/design.md`). A section with no authored date falls back to a *content* date — git history for git-backed layers (`okf-local`, `github`), the file mtime only where no history can exist. Never date a section by mtime in a git-backed layer, and never by committer date: git does not preserve mtimes (a fresh clone would claim every doc was written today) and `pull --rebase` rewrites committer dates (which would re-date the whole bundle on the flow team-sync depends on). Where git genuinely cannot date a tracked file — a shallow clone's boundary commit lists the entire tree — leave the section undated rather than borrowing a date that reads as fresh. - **The manifest is a trust boundary.** An `mcp` layer spawns `command` with `args` from the manifest — a manifest you did not author can run arbitrary commands as your user. Only point `--manifest` at configs you trust (same model as any MCP client config). - **A manifest names credentials, never carries them.** A remote layer's `auth` may only be `"keychain:"` (resolved from the `tokens` map the caller injects into `buildSources` — the app owns the keychain, the engine never opens it) or `{"tokenEnv": "NAME"}` for headless runs. Any other shape throws, which is what structurally keeps a raw token out of a manifest. An alias with no injected secret reads anonymously rather than failing. - **A manifest still decides where a named credential is *sent*.** A `github` layer's `apiBase` (the GitHub Enterprise knob) plus a valid `auth` alias means an untrusted manifest can direct a real token at a host it chooses. This is inside the existing manifest trust boundary, not a separate one — but when the desktop app writes manifests it should only ever emit the default `apiBase`. diff --git a/packages/core/src/sources/files.mjs b/packages/core/src/sources/files.mjs index 758a539..cce7035 100644 --- a/packages/core/src/sources/files.mjs +++ b/packages/core/src/sources/files.mjs @@ -8,7 +8,7 @@ import fs from "node:fs"; import path from "node:path"; -import { parseConcept, parseHeadingAttrs, normalizeConceptId, normalizeHeading } from "./okf-local.mjs"; +import { parseConcept, parseHeadingAttrs, normalizeConceptId, normalizeHeading, withDocumentDate, localDate } from "./okf-local.mjs"; // loadConcept resolution order on id collision (e.g. notes.md + notes.txt). const EXTENSIONS = [".md", ".mdx", ".txt"]; @@ -49,7 +49,7 @@ export function createFilesSource({ name, level, root }) { function parseFile(filePath, ext) { const content = fs.readFileSync(filePath, "utf8"); - const mtime = fs.statSync(filePath).mtime.toISOString().slice(0, 10); + const mtime = localDate(fs.statSync(filePath).mtime); return parseDocument({ content, stem: path.basename(filePath, ext), updated: mtime, ext }); } @@ -58,21 +58,10 @@ function parseFile(filePath, ext) { // mtime) so a GitHub-hosted CLAUDE.md and a local one resolve identically. export function parseDocument({ content, stem, updated, ext = ".md" }) { if (ext === ".txt") return parsePlainText(content, stem, updated); - if (hasFrontmatter(content)) { - const parsed = parseConcept(content); - // Explicit per-section dates remain authoritative. Otherwise an OKF-level - // date wins, then the adapter's document date (mtime or remote commit). - // Without this fallback, structured remote docs silently lose the - // per-document staleness metadata that plain documents receive. - const fallback = parsed.frontmatter.updated ?? updated; - return { - ...parsed, - sections: parsed.sections.map((section) => ({ - ...section, - updated: section.updated ?? fallback, - })), - }; - } + // Undated sections inherit the OKF-level date, then this adapter's document + // date — same rule okf-local applies, so structured docs never lose the + // per-document staleness metadata that plain documents receive. + if (hasFrontmatter(content)) return withDocumentDate(parseConcept(content), updated); return parsePlainMarkdown(content, stem, updated); } diff --git a/packages/core/src/sources/git-core.mjs b/packages/core/src/sources/git-core.mjs index 89e909a..fda7bdf 100644 --- a/packages/core/src/sources/git-core.mjs +++ b/packages/core/src/sources/git-core.mjs @@ -38,7 +38,9 @@ export async function runGit(root, args, { allowFailure = false } = {}) { return { ok: true, stdout: stdout.trim(), stderr: stderr.trim() }; } catch (error) { if (allowFailure) { - return { ok: false, stdout: scrub(error.stdout ?? ""), stderr: scrub(error.stderr ?? error.message) }; + // `||`, not `??`: a maxBuffer/timeout kill leaves stderr an empty string, + // and a caller that surfaces it would report a blank reason. + return { ok: false, stdout: scrub(error.stdout ?? ""), stderr: scrub(error.stderr || error.message) }; } const wrapped = new Error(`git ${args[0]} failed: ${scrub(error.stderr || error.message)}`); wrapped.code = "GitError"; diff --git a/packages/core/src/sources/okf-local.mjs b/packages/core/src/sources/okf-local.mjs index 0d322dc..573b357 100644 --- a/packages/core/src/sources/okf-local.mjs +++ b/packages/core/src/sources/okf-local.mjs @@ -2,12 +2,125 @@ // frontmatter) from disk. Owns all OKF parsing. Implements the source contract: // loadConcept(id) -> { frontmatter, sections } | null // listConceptIds() -> string[] -// close() -> noop +// sync() -> drops the commit-date memo close() -> noop import fs from "node:fs"; import path from "node:path"; +import { runGit } from "./git-core.mjs"; + +// A read-path memo of the layer's git history must not outlive the history — +// mcp-server is a session-lifetime process, so a boot-time snapshot would serve +// stale dates all session. Short enough to pick up local commits, long enough +// that a search sweep is one `git log`, not one per concept. +const HISTORY_TTL_MS = 30000; export function createOkfLocalSource({ name, level, root }) { + let history = null; // { at, promise } — see commitHistory + let warned = false; + + function warnOnce(message) { + if (warned) return; + warned = true; + console.error(`[okf-local source "${name}"] ${message}`); // stderr: stdout is the MCP protocol + } + + // Memoize the PROMISE, never the map. Publishing a half-built map before the + // await would let every concurrent reader find it empty, miss, and fall back + // to the mtime — which is the precise wrong answer this whole path exists to + // prevent, and mcp-server handles requests concurrently. + function commitHistory() { + if (!history || Date.now() - history.at >= HISTORY_TTL_MS) { + history = { at: Date.now(), promise: readHistory() }; + } + return history.promise; + } + + // A section date has to say when the CONTENT last changed. An okf-local + // bundle is a git repo, and git does not preserve mtimes — clone, checkout + // and pull all stamp the working tree with the operation time — so an mtime + // would report "today" for a decision written years ago, in the one field the + // product sells as the staleness signal. Read the commit history instead. + // + // Batched per generation, not per file: mcp-server sweeps loadConcept over + // every id for search/list, so a per-file spawn is not affordable. + async function readHistory() { + const dates = new Map(); + // --git-path resolves even when the git dir is not ".git" (worktrees, GIT_DIR). + const where = await runGit(root, ["rev-parse", "--git-path", "shallow"], { allowFailure: true }); + if (!where.ok) { + // A plain folder is a legitimate okf-local root, so "not a repository" is + // not worth a word. Anything else — no git on PATH, a safe.directory + // refusal — means dates silently degrade to mtimes, and that must be said + // out loud or the layer just quietly starts claiming everything is new. + if (!/not a git repository/i.test(where.stderr)) { + warnOnce(`cannot read git history (${where.stderr}) — sections will fall back to file mtimes`); + } + return { dates, inRepo: false, tracked: null }; + } + const boundary = readShallowBoundary(path.resolve(root, where.stdout)); + + // %as is the AUTHOR date, not the committer date: git-core runs + // `pull --rebase` as its standard divergence recovery, and a rebase rewrites + // every committer date to now — which would re-date the whole bundle to + // today on the very flow team-sync depends on. Author dates survive it. + // --relative prints paths relative to this layer root, so a layer inside a + // bigger repo needs no prefix arithmetic; `-- .` bounds the walk to it. + // core.quotePath=false keeps non-ASCII paths literal so they match our keys. + const log = await runGit( + root, + ["-c", "core.quotePath=false", "log", "--format=%x00%as%x00%H", "--name-only", "--relative", "--", "."], + { allowFailure: true }, + ); + // Inside a repo, a failed history read means the dates are unknown — NOT + // that the mtime is now trustworthy. Returning an empty tracked set here + // would date every doc today, the exact lie this path exists to prevent. + if (!log.ok) { + warnOnce(`cannot read commit dates (${log.stderr}) — sections in this layer will be undated`); + return { dates, inRepo: true, tracked: null }; + } + // Newest-first, so the first date seen for a path is its latest. + let date = null; + for (const line of log.stdout.split("\n")) { + if (line.startsWith("\0")) { + const [, authored, hash] = line.split("\0"); + // A shallow clone's boundary commit lists the ENTIRE tree as added at + // the truncation date. Trusting it would date every untouched file to + // the clone — mtime's lie by another route. Leave those paths undated. + date = boundary.has(hash) ? null : authored; + } else if (line !== "" && date && !dates.has(line)) { + dates.set(line, date); + } + } + // ls-files prints cwd-relative paths, matching the log's --relative keys. + // Needed to tell "untracked, so the mtime is the true edit time" apart from + // "tracked but the history could not date it", where the mtime is a lie. + const listed = await runGit(root, ["-c", "core.quotePath=false", "ls-files"], { allowFailure: true }); + if (!listed.ok) { + warnOnce(`cannot list tracked files (${listed.stderr}) — undatable sections will be undated`); + } + return { dates, inRepo: true, tracked: listed.ok ? new Set(listed.stdout.split("\n").filter(Boolean)) : null }; + } + + async function documentDate(filePath) { + const { dates, inRepo, tracked } = await commitHistory(); + const rel = toPosix(path.relative(root, filePath)); + const authored = dates.get(rel); + if (authored) return authored; + // Inside a repo with no date for this path, the mtime is a checkout time, + // not an edit time. That covers a truncated history (shallow clone), a change + // that only landed inside a merge commit, and a history we could not read at + // all (tracked === null, so we cannot rule any of it out). None of them has + // an honest date — say nothing rather than claim today. + if (inRepo && (tracked === null || tracked.has(rel))) return null; + // Untracked, or no repo at all: here the mtime IS the real edit time, the + // same signal files.mjs uses for the plain folders it points at. + try { + return localDate(fs.statSync(filePath).mtime); + } catch { + return null; // vanished between read and stat — undated beats crashing the resolve + } + } + return { name, level, @@ -15,17 +128,59 @@ export function createOkfLocalSource({ name, level, root }) { const safeId = normalizeConceptId(id); const filePath = path.join(root, `${safeId}.md`); if (!fs.existsSync(filePath)) return null; - return parseConcept(fs.readFileSync(filePath, "utf8")); + const parsed = parseConcept(fs.readFileSync(filePath, "utf8")); + return withDocumentDate(parsed, await documentDate(filePath)); }, async listConceptIds() { return walkMarkdown(root).map((filePath) => toPosix(path.relative(root, filePath)).replace(/\.md$/i, ""), ); }, + // withGitSync calls this after a pull that changed something — new commits + // mean new dates, so the memo must not outlive them. + sync() { + history = null; + }, close() {}, }; } +// ---- document dates -------------------------------------------------------- + +// The commits a shallow clone truncated at. Their diff is the whole tree, so +// their date describes the clone, not the content. +function readShallowBoundary(shallowFile) { + try { + return new Set(fs.readFileSync(shallowFile, "utf8").split("\n").filter(Boolean)); + } catch { + return new Set(); // absent = a complete clone, the common case + } +} + +// Fills in section dates the document did not state. Explicit per-section attrs +// ({updated=}) stay authoritative; otherwise an OKF-level `updated` wins, then +// the adapter's document date (a commit date here and in github.mjs, an mtime +// for the plain folders files.mjs reads). Every adapter applies this — the same +// bytes must produce the same staleness metadata no matter which kind of layer +// read them. Kept out of parseConcept so callers that only want the literal +// document (promote.mjs, team-activity.mjs) are unaffected. +export function withDocumentDate(parsed, documentDate) { + const fallback = parsed.frontmatter.updated ?? documentDate ?? null; + return { + ...parsed, + sections: parsed.sections.map((section) => ({ ...section, updated: section.updated ?? fallback })), + }; +} + +// A file's mtime is a local wall-clock event, so it has to be formatted in +// local time: toISOString() would file an edit made this evening under +// tomorrow's date. (API timestamps are real UTC instants — github.mjs formats +// those as UTC, correctly.) +export function localDate(when) { + const pad = (value) => String(value).padStart(2, "0"); + return `${when.getFullYear()}-${pad(when.getMonth() + 1)}-${pad(when.getDate())}`; +} + // ---- OKF parsing (moved verbatim from resolver.mjs) ------------------------ export function parseConcept(content) { diff --git a/packages/core/tests/files-source-test.sh b/packages/core/tests/files-source-test.sh index 63b708e..a73f148 100755 --- a/packages/core/tests/files-source-test.sh +++ b/packages/core/tests/files-source-test.sh @@ -74,7 +74,11 @@ else console.log(JSON.stringify(await s.loadConcept(process.argv[3]))); EOF # Section dates come from file mtime — compute the expected date from the file itself. -guide_date="$(node -e 'console.log(require("node:fs").statSync(process.argv[1]).mtime.toISOString().slice(0,10))' "$docs/guide.md")" +# These fixtures were written moments ago, so their mtime date is today's LOCAL +# date. Deriving it from `date` rather than from the adapter's own formula is +# what makes this able to catch a UTC-vs-local slip: an evening run under +# toISOString() reports tomorrow, and this comparison fails. +guide_date="$(date +%Y-%m-%d)" # --- 1. Plain .md: synthesized frontmatter, okf-normalized keys, mtime dates -- @@ -108,6 +112,39 @@ grep -q '"key":"engine"' <<<"$okf_out" || fail "OKF {#key} attr should be honore grep -q '"updated":"2026-04-01"' <<<"$okf_out" || fail "OKF updated= attr should be honored" "$okf_out" grep -q '"override":"none"' <<<"$okf_out" || fail "OKF override= attr should be honored" "$okf_out" +# --- 2b. Adapter parity: same bytes, same dates, whichever adapter reads them -- +# An OKF doc with no `updated:` anywhere used to resolve to the mtime through a +# files layer and to null through an okf-local layer. $docs is a plain folder, +# not a git repo, so okf-local has no commit date to prefer and lands on the +# same mtime files.mjs uses. The git-backed case — where the two legitimately +# differ, because only one of them can see the real authorship date — is +# resolver-test.sh's commit-date step. + +cat > "$docs/undated.md" <<'EOF' +--- +type: decision +title: Undated doc +--- + +## Engine {#engine} + +Postgres. +EOF + +cat > "$tmpdir/parity.mjs" < "$gitrepo/decisions/legacy.md" <<'EOF' +--- +type: decision +title: Legacy decision +--- + +## Engine + +MySQL. Written years ago, never re-dated. +EOF +# A layer rooted at a subdirectory of the repo: git reports paths from the repo +# root, so this is the shape that silently falls back to mtime if the adapter +# forgets to account for its own prefix. +cp "$gitrepo/decisions/legacy.md" "$gitrepo/nested/decisions/legacy.md" +git -C "$gitrepo" init -q . +git -C "$gitrepo" config user.email test@example.com +git -C "$gitrepo" config user.name Test +git -C "$gitrepo" add -A +# Author date 2019, committer date today — the shape `git pull --rebase` leaves +# behind, and git-core runs exactly that as its divergence recovery. Reading the +# committer date would re-date the whole bundle to today on the flow team-sync +# depends on, so the author date is the one that must win. +GIT_COMMITTER_DATE="2026-01-15T12:00:00" git -C "$gitrepo" commit -q --date="2019-03-01T12:00:00" -m "the original decision" +# What a clone/checkout does to the mtime. +touch "$gitrepo/decisions/legacy.md" "$gitrepo/nested/decisions/legacy.md" + +# An uncommitted sibling: no commit date exists, and there the mtime IS the real +# edit time — the same signal a files layer uses for a plain folder. +cp "$gitrepo/decisions/legacy.md" "$gitrepo/decisions/uncommitted.md" + +cat > "$tmpdir/git-layers.json" <<'EOF' +{ "layers": [ { "name": "team", "level": 2, "path": "gitrepo" } ] } +EOF +cat > "$tmpdir/git-nested-layers.json" <<'EOF' +{ "layers": [ { "name": "team", "level": 2, "path": "gitrepo/nested" } ] } +EOF + +dated="$(node "$resolver" --manifest "$tmpdir/git-layers.json" --concept decisions/legacy)" +grep -q '"sourceUpdated": "2019-03-01"' <<<"$dated" || fail "an undated section in a git bundle should carry the last-commit date, not the mtime" "$dated" + +nested="$(node "$resolver" --manifest "$tmpdir/git-nested-layers.json" --concept decisions/legacy)" +grep -q '"sourceUpdated": "2019-03-01"' <<<"$nested" || fail "a layer rooted in a subdirectory of a repo should still get commit dates" "$nested" + +untracked="$(node "$resolver" --manifest "$tmpdir/git-layers.json" --concept decisions/uncommitted)" +grep -q "\"sourceUpdated\": \"$(date +%Y-%m-%d)\"" <<<"$untracked" || fail "an uncommitted doc has no commit date and should fall back to its mtime" "$untracked" + +# Concurrent reads must agree with sequential ones. mcp-server does not await its +# request handler, so reads overlap; a history memo published before its own +# await would leave every concurrent reader with the mtime instead. +cat > "$tmpdir/concurrent.mjs" < source.loadConcept(id))); +console.log(JSON.stringify(loaded.map((c) => c.sections[0].updated))); +EOF +concurrent="$(node "$tmpdir/concurrent.mjs")" +[ "$concurrent" = '["2019-03-01","2019-03-01","2019-03-01"]' ] || fail "concurrent loads should all get the commit date, not the mtime" "$concurrent" + +# A shallow clone's boundary commit lists the whole tree as added at the +# truncation date. Dating from it would claim every file was written at clone +# time — mtime's lie by another route — so those sections stay undated. +git clone -q --depth 1 "file://$gitrepo" "$tmpdir/shallow" +cat > "$tmpdir/shallow-layers.json" <<'EOF' +{ "layers": [ { "name": "team", "level": 2, "path": "shallow" } ] } +EOF +shallow="$(node "$resolver" --manifest "$tmpdir/shallow-layers.json" --concept decisions/legacy)" +grep -q '"sourceUpdated": null' <<<"$shallow" || fail "a shallow clone cannot date its history and must say so, not invent the boundary date" "$shallow" + +# A git that fails inside a real repo means the dates are unknown — it does not +# make the mtime trustworthy. Falling back to it here would date every doc today, +# which is the failure this whole path exists to prevent. +mkdir -p "$tmpdir/shim" +cat > "$tmpdir/shim/git" <&2; exit 128; fi +exec "$(command -v git)" "\$@" +EOF +chmod +x "$tmpdir/shim/git" +broken="$(PATH="$tmpdir/shim:$PATH" node "$resolver" --manifest "$tmpdir/git-layers.json" --concept decisions/legacy 2>/dev/null)" +grep -q '"sourceUpdated": null' <<<"$broken" || fail "an unreadable git history should leave sections undated, not fall back to the mtime" "$broken" + # --- Path-traversal guard: a concept id must not escape its layer root --- for evil in ".." "../secrets" "decisions/../../etc/passwd" "a/.." "/etc/passwd"; do if node "$resolver" --manifest "$tmpdir/conf-layers.json" --concept "$evil" 2>/dev/null; then @@ -151,4 +241,4 @@ for evil in ".." "../secrets" "decisions/../../etc/passwd" "a/.." "/etc/passwd"; fi done -echo "resolver test passed (section merge + provenance + vertical precedence + suppression + conflicts + traversal guard)" +echo "resolver test passed (section merge + provenance + vertical precedence + suppression + conflicts + commit dates + traversal guard)" diff --git a/specs/contextcake-integrations/spec.md b/specs/contextcake-integrations/spec.md index 45d4ecb..60b9d10 100644 --- a/specs/contextcake-integrations/spec.md +++ b/specs/contextcake-integrations/spec.md @@ -51,6 +51,7 @@ serves and marks staleness. | Kind | Concept id | Sections | `updated` | Auth (read-only scopes) | |---|---|---|---|---| +| `okf-local` | relative path minus `.md` | OKF frontmatter and heading attrs, authoritative | OKF attr, then frontmatter `updated`, then the file's last **author** date (committer dates do not survive `pull --rebase`). File mtime only where no history can exist — an untracked doc, or a root that is not a repo. A tracked doc git cannot date (shallow clone) stays undated rather than borrowing the clone's date | none | | `files` | relative path minus extension | OKF frontmatter honored when present; plain markdown → `##` headings (keys via okf-local's `normalizeHeading`, so sections merge across adapter kinds), preamble → `overview`; `.txt` → `body` | file mtime unless OKF attr | none | | `github` | `//` within the layer | same plain-markdown rules as `files` | latest commit date for the file (cached; repo `pushed_at` fallback) | GitHub App **device flow**, `contents:read` | | `slack` | `/` + one `/channel` concept (topic/purpose) | canvas headings → sections; a pinned message → `body` | message/canvas edit timestamp | Slack app user token: `channels:read`, `pins:read`, `canvases:read` |