From f7891f5ee75388f1b4e0b0c2692dd6775813f08f Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:03:20 -0400 Subject: [PATCH 01/28] feat(cache): make persisted indexes portable --- AGENTS.md | 2 +- codegraph-skill/codegraph/SKILL.md | 2 +- docs/cli.md | 3 + src/cli/doctor.ts | 13 +- src/cli/help.ts | 5 +- src/cli/invocationContext.ts | 3 + src/cli/options.ts | 4 +- src/config.ts | 34 ++- src/graphs/symbol-graph-detailed.ts | 4 +- src/indexer/build-cache.ts | 1 + src/indexer/build-cache/manifest.ts | 51 +++- src/indexer/build-cache/module-cache.ts | 151 ++++++++++-- src/indexer/build-cache/options.ts | 15 +- src/indexer/build-cache/project-snapshot.ts | 257 +++++++++++++++----- src/indexer/build-index.ts | 7 +- src/indexer/build-manifest.ts | 5 +- src/indexer/finalize.ts | 8 +- src/indexer/navigation.ts | 9 +- src/indexer/parse-context.ts | 3 +- src/indexer/types.ts | 4 + 20 files changed, 470 insertions(+), 111 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 75497aff..aa943103 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ - For repo-understanding flows, start with `node ./dist/cli.js doctor` and `node ./dist/cli.js orient --root . --budget small --json` when `dist` is built; build first if validating the working tree from a fresh checkout. - For source-checkout validation and contributor examples, prefer `node ./dist/cli.js ...`; reserve bare `codegraph ...` for published/global install guidance. - When package metadata, install scripts, optional native dependencies, or the resolved npm graph changes, update `package-lock.json` in the same change and verify with `npm ci --ignore-scripts --dry-run` unless lifecycle scripts are part of the behavior under test. -- Treat `--root` as the project boundary for config lookup, cache/manifests, path confinement, and output normalization. When `--root` is set, positional paths are include roots; for `orient` and `drift`, positional paths are always include roots. +- Treat `--root` as the project boundary for config lookup, path confinement, and output normalization. Cache/manifests may use the resolved cache anchor (`--cache-dir`/`CODEGRAPH_CACHE_DIR`, repository metadata, or project root); cached contents remain project-relative. - Keep discovery glob guidance accurate: `codegraph.config.json` globs are project-root-relative, while CLI `--include-glob`/`--ignore-glob` values are one-off filters relative to each active scan root. - Within any claimed cross-language capability, behavior should stay consistent across all supported languages for that capability. Avoid language-subset branches; if a limitation is intentional, document it in the parity docs and cover it with explicit tests in the same change. - When language support changes, update `docs/language-parity.md` and `docs/scenario-catalog.md` in the same change so support claims, limitations, and fixture coverage stay aligned. diff --git a/codegraph-skill/codegraph/SKILL.md b/codegraph-skill/codegraph/SKILL.md index 6b306580..3f2c1868 100644 --- a/codegraph-skill/codegraph/SKILL.md +++ b/codegraph-skill/codegraph/SKILL.md @@ -34,7 +34,7 @@ Prefer `review` before `impact`: review is the compact reviewer handoff; impact ## Keep the Project Boundary Explicit -Use `--root` to define the boundary for config lookup, cache scope, path confinement, and output normalization. +Use `--root` to define the boundary for config lookup, path confinement, and output normalization. Cache contents use project-relative paths and may live at the resolved repository anchor; override location with `--cache-dir` or `CODEGRAPH_CACHE_DIR`. - Positional paths are include roots inside the project boundary for `orient`, `drift`, and positional graph commands. - `codegraph.config.json` discovery globs are project-root-relative. diff --git a/docs/cli.md b/docs/cli.md index b49e01dc..8758f4a7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -21,6 +21,9 @@ The `graph` command without output-format flags writes Mermaid to stdout. Use `- Numeric options such as `--limit`, `--threads`, `--depth`, `--max-refs`, and token bounds must be integers in their documented ranges; invalid numeric values fail instead of being silently clamped or ignored. Default workflow: +## Cache location + +Index caches store project-relative paths, so a cache can be moved with its project. Cache selection precedence is `--cache-dir`, `CODEGRAPH_CACHE_DIR`, `cache.location` in project config (then user config), repository metadata, then the project root. `cache.location` accepts `project`, `repo`, `user`, or an absolute path; `--root` remains the project scope boundary. - code review: `codegraph review` - blast-radius follow-up: `codegraph impact --base HEAD --head WORKTREE` diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index be04890f..ead0abe6 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { cacheRoot, resolveCacheAnchor } from "../indexer/build-cache/module-cache.js"; import { isNativeTreeSitterAvailable, getNativeBindingOrigin, @@ -34,7 +35,6 @@ export type DoctorNativeUpdateReport = { installedVersion?: string; reason?: string; }; - export type DoctorReport = { package: CodegraphPackageIdentity; native: { @@ -44,6 +44,11 @@ export type DoctorReport = { origin?: DoctorNativeOriginReport; update?: DoctorNativeUpdateReport; }; + cache: { + path: string; + anchor: string; + layer: string; + }; indexArtifact?: IndexedArtifactReport; }; @@ -310,8 +315,14 @@ export function buildDoctorReport(indexPath?: string): DoctorReport { const origin = getNativeBindingOrigin(); const runtimeIdentity = captureCodegraphRuntimeIdentity(origin); const update = createInstalledVersionChecker(runtimeIdentity, { warn: () => undefined }).check(true); + const cacheResolution = resolveCacheAnchor(process.cwd()); return { package: packageIdentity, + cache: { + path: normalizePathForDisplay(cacheRoot(process.cwd())), + anchor: normalizePathForDisplay(cacheResolution.anchor), + layer: cacheResolution.layer, + }, native: { available: isNativeTreeSitterAvailable(), ...(loadError ? { loadError: String(loadError) } : {}), diff --git a/src/cli/help.ts b/src/cli/help.ts index 7eae91cf..22c275b2 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -43,6 +43,7 @@ Build Options: --native Native runtime mode: auto, on, off --workers Force Piscina native-extraction workers (auto above 250 files) --cache Cache mode: disk, memory, off + --cache-dir Cache location override (also CODEGRAPH_CACHE_DIR) --limit N Result limit for hotspots/inspect summaries --cache-strict Force strict content-hash cache validation --cache-verify Re-stat cached files before trusting disk cache entries @@ -178,9 +179,9 @@ Usage: codegraph uninstall [target] [--target { const cache = parseCacheModeOption(getOpt("--cache")); const threads = parseOptionalNonNegativeIntegerOption(getOpt("--threads"), "--threads"); + const cacheDir = getOpt("--cache-dir"); return { ...(base.progressHandler ? { onProgress: base.progressHandler } : {}), discovery: discoveryOptions, ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...(cache !== undefined ? { cache } : {}), + ...(cacheDir ? { cacheDir } : {}), + ...(config.cache?.location ? { cacheLocation: config.cache.location } : {}), ...(hasFlag("--cache-strict") ? { cacheStrict: true } : {}), ...(hasFlag("--cache-verify") ? { cacheVerify: true } : {}), ...(hasGraphOverrides || base.nativeMode !== "auto" ? { graph: base.buildGraphOptions() } : {}), diff --git a/src/cli/options.ts b/src/cli/options.ts index 3673de7c..36934293 100644 --- a/src/cli/options.ts +++ b/src/cli/options.ts @@ -14,8 +14,7 @@ const CLI_VALUE_OPTIONS = new Set([ "--threads", "--native", "--cache", - "--changed-since", - "--git-base", + "--cache-dir", "--git-head", "--symbols-detailed-scope", "--symbols-detailed-max-edges", @@ -127,6 +126,7 @@ const SHARED_BUILD_OPTIONS = [ "--threads", "--native", "--cache", + "--cache-dir", "--include-glob", "--ignore-glob", "--resolution-hint", diff --git a/src/config.ts b/src/config.ts index c5f75e40..13dca7e9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,5 @@ import fsp from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { z } from "zod"; import { @@ -41,12 +42,20 @@ const codegraphConfigSchema = z }) .strict() .optional(), + cache: z + .object({ + location: z.string().trim().min(1), + }) + .optional(), }) .strict(); type ParsedCodegraphConfig = z.infer; export type CodegraphConfig = { + cache?: { + location: string; + }; discovery?: ProjectFileDiscoveryOptions; languages?: { extensions?: LanguageExtensionMap; @@ -147,27 +156,41 @@ function normalizeConfigLanguageExtensions( } return normalizeLanguageExtensions(extensions); } +async function loadUserCacheLocation(): Promise { + const configRoot = + process.platform === "win32" + ? process.env.APPDATA?.trim() || path.join(os.homedir(), "AppData", "Roaming") + : process.env.XDG_CONFIG_HOME?.trim() || path.join(os.homedir(), ".config"); + const configPath = path.join(configRoot, "codegraph", "config.json"); + try { + const parsedJson = JSON.parse(await fsp.readFile(configPath, "utf8")) as unknown; + const parsed = codegraphConfigSchema.safeParse(parsedJson); + if (!parsed.success) throw new Error(z.prettifyError(parsed.error)); + return parsed.data.cache?.location; + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined; + throw new Error(`Invalid user codegraph config: ${errorMessage(error)}`); + } +} export async function loadCodegraphConfig(projectRoot: string): Promise { + const userCacheLocation = await loadUserCacheLocation(); const configPath = path.join(projectRoot, CODEGRAPH_CONFIG_FILE); let raw: string; try { raw = await fsp.readFile(configPath, "utf8"); } catch (error) { if (error instanceof Error && "code" in error && error.code === "ENOENT") { - return {}; + return userCacheLocation ? { cache: { location: userCacheLocation } } : {}; } throw error; } - let parsedJson: unknown; try { parsedJson = JSON.parse(raw); } catch (error) { - const message = errorMessage(error); - throw new Error(`Invalid ${CODEGRAPH_CONFIG_FILE}: ${message}`); + throw new Error(`Invalid ${CODEGRAPH_CONFIG_FILE}: ${errorMessage(error)}`); } - const parsed = codegraphConfigSchema.safeParse(parsedJson); if (!parsed.success) { throw new Error(`Invalid ${CODEGRAPH_CONFIG_FILE}: ${z.prettifyError(parsed.error)}`); @@ -177,6 +200,7 @@ export async function loadCodegraphConfig(projectRoot: string): Promise, + toRelative: boolean, +): Record { + const transformed: Record = {}; + for (const [file, entry] of Object.entries(files)) { + const key = toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file); + transformed[key] = { + ...entry, + edges: entry.edges.map((edge) => ({ + ...edge, + from: toRelative ? cacheRelativePath(projectRoot, edge.from) : cacheAbsolutePath(projectRoot, edge.from), + to: + edge.to.type === "file" + ? { + ...edge.to, + path: toRelative ? cacheRelativePath(projectRoot, edge.to.path) : cacheAbsolutePath(projectRoot, edge.to.path), + } + : edge.to, + })), + }; + } + return transformed; +} type ConfigHashResult = { hash: string; error?: string; @@ -234,16 +263,22 @@ export async function loadManifest(projectRoot: string, opts?: BuildOptions): Pr const raw = await fsp.readFile(manifestPath, "utf8"); const parsed = JSON.parse(raw) as IndexManifest; if ( - parsed.version !== MANIFEST_VERSION || + (parsed.version !== MANIFEST_VERSION && parsed.version !== 3) || typeof parsed.projectRoot !== "string" || !/^[a-f0-9]{64}$/.test(parsed.buildOptions?.implementationFingerprint ?? "") ) { return null; } - const activeRootIdentity = fileIdentityKey(path.resolve(projectRoot)); - const manifestRootIdentity = fileIdentityKey(path.resolve(parsed.projectRoot)); - if (manifestRootIdentity !== activeRootIdentity) return null; - return parsed; + const oldFiles = parsed.files ?? {}; + const relativeFiles = + parsed.version === 3 ? transformManifestEntries(parsed.projectRoot, oldFiles, true) : oldFiles; + const migrated: IndexManifest = { + ...parsed, + version: MANIFEST_VERSION, + files: transformManifestEntries(projectRoot, relativeFiles, false), + transientFiles: sanitizeManifestTransientFilesForRoot(projectRoot, parsed.transientFiles), + }; + return migrated; } catch { return null; } diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 4f89e9bc..5f2a3778 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -2,6 +2,7 @@ import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } import crypto from "node:crypto"; import fs from "node:fs"; import fsp from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { supportForFile } from "../../languages.js"; import { getNativeRuntimeFingerprint } from "../../native/treeSitterNative.js"; @@ -21,9 +22,9 @@ import { lruMapGet, lruMapSet } from "../../util/lruMap.js"; import { initCacheReport } from "./reports.js"; import { getImplementationFingerprint } from "./options.js"; -// v3: implementation fingerprint and root-namespaced custom cache directories. -const PARSED_CACHE_VERSION = 3; -const MODULE_CACHE_SCHEMA_VERSION = 1; +// v4: relative file keys and explicit cache-anchor policy. +const PARSED_CACHE_VERSION = 4; +const MODULE_CACHE_SCHEMA_VERSION = 2; const MODULE_CACHE_TABLE = "module_cache"; const MODULE_CACHE_SCHEMA_VERSION_KEY = "module_cache.schema_version"; const MODULE_CACHE_COLUMNS: readonly SqliteTableColumn[] = [ @@ -83,21 +84,110 @@ function reportMissingNodeSqlite(logLevel: import("../../logging.js").LogLevel | error, ); } - -function projectCacheNamespace(projectRoot: string): string { +export function projectCacheNamespace(projectRoot: string): string { const rootIdentity = fileIdentityKey(path.resolve(projectRoot)); const hash = crypto.createHash("sha256").update(rootIdentity).digest("hex"); return `project-${hash}`; } +export type CacheAnchorResolution = { + anchor: string; + layer: "explicit" | "environment" | "manifest" | "git" | "project" | "user"; +}; + +function isWritableDirectory(candidate: string): boolean { + try { + return fs.statSync(candidate).isDirectory() && fs.accessSync(candidate, fs.constants.W_OK) === undefined; + } catch { + return false; + } +} + +function isForbiddenAnchor(candidate: string): boolean { + const resolved = path.resolve(candidate); + const home = path.resolve(os.homedir()); + const parsed = path.parse(resolved); + return fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root); +} + +function findRepositoryAnchor(projectRoot: string): CacheAnchorResolution { + const start = path.resolve(projectRoot); + let current = start; + while (true) { + const manifest = path.join(current, ".codegraph", "manifest.json"); + if (fs.existsSync(manifest) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { + return { anchor: current, layer: "manifest" }; + } + const gitPath = path.join(current, ".git"); + if (fs.existsSync(gitPath) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { + return { anchor: current, layer: "git" }; + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + return { anchor: start, layer: "project" }; +} + +function resolveCodegraphUserCacheRoot(): string { + const base = + process.platform === "win32" + ? process.env.LOCALAPPDATA?.trim() || path.join(os.homedir(), "AppData", "Local") + : process.env.XDG_CACHE_HOME?.trim() || path.join(os.homedir(), ".cache"); + return path.join(base, "codegraph"); +} +export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): CacheAnchorResolution { + const explicit = opts?.cacheDir?.trim(); + if (explicit) return { anchor: path.resolve(explicit), layer: "explicit" }; + const environment = process.env.CODEGRAPH_CACHE_DIR?.trim(); + if (environment) return { anchor: path.resolve(environment), layer: "environment" }; + const location = opts?.cacheLocation; + if (location === "project") return { anchor: path.resolve(projectRoot), layer: "project" }; + if (location === "user") return { anchor: resolveCodegraphUserCacheRoot(), layer: "user" }; + if (location && location !== "repo") return { anchor: path.resolve(location), layer: "explicit" }; + return findRepositoryAnchor(projectRoot); +} + export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { - if (!opts?.cacheDir) return path.join(projectRoot, ".codegraph-cache", "index-v1"); - const namespace = projectCacheNamespace(projectRoot); - const configuredCacheDir = path.resolve(opts.cacheDir); - if (path.basename(configuredCacheDir) === namespace) return configuredCacheDir; - return path.join(configuredCacheDir, namespace); + const root = path.resolve(projectRoot); + const resolution = resolveCacheAnchor(root, opts); + const anchor = isWritableDirectory(resolution.anchor) ? resolution.anchor : root; + const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); + if ( + sameRoot && + !opts?.cacheDir && + !process.env.CODEGRAPH_CACHE_DIR?.trim() && + (!opts?.cacheLocation || opts.cacheLocation === "project") + ) { + return path.join(root, ".codegraph-cache", "index-v1"); + } + const namespace = projectCacheNamespace(root); + const explicitBase = opts?.cacheDir?.trim() || process.env.CODEGRAPH_CACHE_DIR?.trim(); + if (explicitBase) { + const configured = path.resolve(explicitBase); + if (path.basename(configured) === namespace) return configured; + return path.join(configured, namespace); + } + const base = opts?.cacheLocation === "user" + ? resolveCodegraphUserCacheRoot() + : path.join(anchor, ".codegraph-cache"); + const candidate = path.join(path.resolve(base), "index-v1", namespace); + if (!sameRoot && !opts?.cacheLocation) { + const legacy = path.join(root, ".codegraph-cache", "index-v1"); + if (!fs.existsSync(candidate) && fs.existsSync(legacy)) return legacy; + } + return candidate; +} +export function cacheRelativePath(projectRoot: string, file: string): string { + const root = path.resolve(projectRoot); + const absolute = path.isAbsolute(file) ? file : path.resolve(root, file); + const relative = path.relative(root, absolute).replace(/\\/g, "/"); + return relative || "."; } +export function cacheAbsolutePath(projectRoot: string, file: string): string { + return path.isAbsolute(file) ? normalizePath(file) : normalizePath(path.resolve(projectRoot, file)); +} export function cacheDatabasePath(projectRoot: string, opts: BuildOptions | undefined, filename: string): string { return path.join(cacheRoot(projectRoot, opts), filename).replace(/\\/g, "/"); } @@ -114,7 +204,7 @@ function recreateModuleCacheTable(db: SqliteDatabase): void { recreateSqliteTable(db, MODULE_CACHE_TABLE, createModuleCacheTable); } -function migrateModuleCacheTable(db: SqliteDatabase): void { +function migrateModuleCacheTable(db: SqliteDatabase, projectRoot: string): void { const columns = sqliteTableColumns(db, MODULE_CACHE_TABLE); if (!columns.size) { createModuleCacheTable(db); @@ -128,16 +218,22 @@ function migrateModuleCacheTable(db: SqliteDatabase): void { if (!columns.has("version")) db.exec("ALTER TABLE module_cache ADD COLUMN version INTEGER NOT NULL DEFAULT 0;"); if (!columns.has("payload")) db.exec("ALTER TABLE module_cache ADD COLUMN payload TEXT NOT NULL DEFAULT '{}';"); if (!columns.has("updated_at")) db.exec("ALTER TABLE module_cache ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0;"); + const rows = db.prepare("SELECT file FROM module_cache").all() as Array<{ file: string }>; + const update = db.prepare("UPDATE module_cache SET file = ? WHERE file = ?"); + for (const row of rows) { + const relative = cacheRelativePath(projectRoot, row.file); + if (relative !== row.file) update.run(relative, row.file); + } } -function ensureModuleCacheSchema(db: SqliteDatabase): void { +function ensureModuleCacheSchema(db: SqliteDatabase, projectRoot: string): void { ensureSqliteVersionedTableSchema({ db, tableName: MODULE_CACHE_TABLE, schemaVersionKey: MODULE_CACHE_SCHEMA_VERSION_KEY, schemaVersion: MODULE_CACHE_SCHEMA_VERSION, createTable: createModuleCacheTable, - migrateTable: migrateModuleCacheTable, + migrateTable: (database) => migrateModuleCacheTable(database, projectRoot), }); db.exec("CREATE INDEX IF NOT EXISTS idx_module_cache_sig ON module_cache(sig);"); } @@ -158,7 +254,7 @@ function getDiskModuleCache(projectRoot: string, opts?: BuildOptions): DiskModul } db.pragma("journal_mode = WAL"); db.pragma("synchronous = NORMAL"); - ensureModuleCacheSchema(db); + ensureModuleCacheSchema(db, projectRoot); db.exec("CREATE TEMP TABLE IF NOT EXISTS live_module_cache_files(file TEXT PRIMARY KEY) WITHOUT ROWID;"); const cache: DiskModuleCache = { db, @@ -206,7 +302,7 @@ export function pruneDiskModuleCache(projectRoot: string, liveFiles: Iterable { cache.clearLiveFiles.run(); - for (const file of liveFiles) cache.insertLiveFile.run(file); + for (const file of liveFiles) cache.insertLiveFile.run(cacheRelativePath(projectRoot, file)); const result = cache.pruneStaleFiles.run(); cache.clearLiveFiles.run(); return Number(result.changes); @@ -335,6 +431,20 @@ function isModuleIndex(value: unknown): value is ModuleIndex { ); } +function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelative: boolean): ModuleIndex { + const copy = structuredClone(module); + const transform = (file: string): string => (toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file)); + copy.file = transform(copy.file); + for (const local of copy.locals) local.file = transform(local.file); + for (const entry of copy.exports) { + if (entry.type === "local") entry.target.file = transform(entry.target.file); + } + for (const binding of copy.imports) { + if (typeof binding.resolved === "string") binding.resolved = transform(binding.resolved); + } + return copy; +} + export function tryLoadFromCache( projectRoot: string, file: string, @@ -362,12 +472,14 @@ export function tryLoadFromCache( if (mode === "disk") { try { const cache = getDiskModuleCache(projectRoot, opts); - const row = cache.load.get(file) as { sig: string; version: number; payload: Uint8Array } | undefined; + const row = cache.load.get(cacheRelativePath(projectRoot, file)) as + | { sig: string; version: number; payload: Uint8Array } + | undefined; if (row && row.sig === sig && row.version === PARSED_CACHE_VERSION) { const parsed: unknown = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")); if (isModuleIndex(parsed)) { if (cacheEnabled && cacheReport) cacheReport.hits += 1; - return parsed; + return transformModulePaths(projectRoot, parsed, false); } } } catch (error) { @@ -375,7 +487,6 @@ export function tryLoadFromCache( reportMissingNodeSqlite(opts?.logLevel, error); return null; } - // cache read failed } if (cacheEnabled && cacheReport) cacheReport.misses += 1; } @@ -400,10 +511,10 @@ export function writeToCache( } else if (mode === "disk") { try { const cache = getDiskModuleCache(projectRoot, opts); - const payload = brotliCompressSync(JSON.stringify(mod), { + const payload = brotliCompressSync(JSON.stringify(transformModulePaths(projectRoot, mod, true)), { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, }); - cache.write.run(file, sig, PARSED_CACHE_VERSION, payload, Date.now()); + cache.write.run(cacheRelativePath(projectRoot, file), sig, PARSED_CACHE_VERSION, payload, Date.now()); } catch (error) { if (isNodeSqliteUnavailableError(error)) { reportMissingNodeSqlite(opts?.logLevel, error); diff --git a/src/indexer/build-cache/options.ts b/src/indexer/build-cache/options.ts index 3c2e282c..16840f09 100644 --- a/src/indexer/build-cache/options.ts +++ b/src/indexer/build-cache/options.ts @@ -9,6 +9,9 @@ import { getCodegraphVersion } from "../../util/packageInfo.js"; import { type ProjectFileDiscoveryOptions } from "../../util/projectFiles.js"; import { getNativeRuntimeFingerprint } from "../../native/treeSitterNative.js"; import type { BuildOptions } from "../types.js"; +export { normalizeLanguageExtensions } from "../../languages.js"; + +export const CORE_ALGORITHM_EPOCH = 2; export type ManifestBuildOptions = { cache?: BuildOptions["cache"]; @@ -17,6 +20,7 @@ export type ManifestBuildOptions = { incrementalStrict?: boolean; nativeRuntimeFingerprint?: string; implementationFingerprint?: string; + coreAlgorithmEpoch?: number; discovery?: { includeGlobs?: string[]; ignoreGlobs?: string[]; @@ -152,7 +156,9 @@ export function getImplementationFingerprint(): string { .map(languageDefinitionFingerprintDescriptor) .sort((left, right) => left.id.localeCompare(right.id)); const hash = crypto.createHash("sha256"); - hash.update("codegraph-implementation-fingerprint-v1"); + hash.update("codegraph-implementation-fingerprint-v2"); + hash.update("\0"); + hash.update(String(CORE_ALGORITHM_EPOCH)); hash.update("\0"); hash.update(getCodegraphVersion()); hash.update("\0"); @@ -174,6 +180,7 @@ function normalizeManifestBuildOptions(opts?: ManifestBuildOptions): ManifestBui incrementalStrict: opts?.incrementalStrict ?? false, ...(opts?.nativeRuntimeFingerprint ? { nativeRuntimeFingerprint: opts.nativeRuntimeFingerprint } : {}), ...(opts?.implementationFingerprint ? { implementationFingerprint: opts.implementationFingerprint } : {}), + coreAlgorithmEpoch: opts?.coreAlgorithmEpoch ?? 1, ...(opts?.discovery ? { discovery: opts.discovery } : {}), ...(languageExtensions ? { languageExtensions } : {}), }; @@ -199,8 +206,6 @@ function normalizeDiscoveryOptions(discovery?: ProjectFileDiscoveryOptions): Man }; } -export { normalizeLanguageExtensions } from "../../languages.js"; - function normalizeBuildOptions(opts?: BuildOptions): ManifestBuildOptions { const discovery = normalizeDiscoveryOptions(opts?.discovery); const languageExtensions = normalizeLanguageExtensions(opts?.languageExtensions); @@ -211,6 +216,7 @@ function normalizeBuildOptions(opts?: BuildOptions): ManifestBuildOptions { incrementalStrict: opts?.incrementalStrict ?? false, nativeRuntimeFingerprint: getNativeRuntimeFingerprint(opts?.native), implementationFingerprint: getImplementationFingerprint(), + coreAlgorithmEpoch: CORE_ALGORITHM_EPOCH, ...(discovery ? { discovery } : {}), ...(languageExtensions ? { languageExtensions } : {}), }; @@ -289,6 +295,9 @@ export function diffBuildOptions( if (normalizedManifest.nativeRuntimeFingerprint !== normalizedCurrent.nativeRuntimeFingerprint) { diffs.push("native"); } + if ((normalizedManifest.coreAlgorithmEpoch ?? 1) !== (normalizedCurrent.coreAlgorithmEpoch ?? CORE_ALGORITHM_EPOCH)) { + diffs.push("coreAlgorithm"); + } if (normalizedManifest.implementationFingerprint !== normalizedCurrent.implementationFingerprint) { diffs.push("implementation"); } diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index aa4a33cc..41906a6a 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -31,18 +31,17 @@ import { type SymbolVisibility, } from "../../graphs/symbol-graph.js"; import { getImplementationFingerprint, normalizeGraphOptions } from "./options.js"; -import { cacheRoot } from "./module-cache.js"; +import { cacheAbsolutePath, cacheRelativePath, cacheRoot } from "./module-cache.js"; import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); -const PROJECT_SNAPSHOT_VERSION = 4; +const PROJECT_SNAPSHOT_VERSION = 5; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; const BLOOM_FILTER_MAX_HASH_COUNT = 10; -const DETAILED_SYMBOL_GRAPH_SNAPSHOT_VERSION = 2; +const DETAILED_SYMBOL_GRAPH_SNAPSHOT_VERSION = 3; const DETAILED_SYMBOL_GRAPH_SNAPSHOT_FILENAME = "detailed-symbol-graph.json"; - const SNAPSHOT_TEMP_RETENTION_MS = 24 * 60 * 60 * 1_000; const SNAPSHOT_TEMP_SUFFIX = ".tmp"; const MAX_SNAPSHOT_CACHE_ENTRIES = 32; @@ -104,6 +103,7 @@ type ProjectIndexSnapshotPayload = { }; modules: ModuleIndex[]; projectRoot: string; + languageExtensions?: ProjectIndex["languageExtensions"]; nativeMode?: ProjectIndex["nativeMode"]; nativeRuntimeFingerprint: string; implementationFingerprint: string; @@ -113,10 +113,13 @@ type ProjectIndexSnapshotPayload = { analysisReport?: SnapshotAnalysisReport; }; -export function projectSnapshotFilesSignature(entries: ReadonlyMap): string { +export function projectSnapshotFilesSignature( + entries: ReadonlyMap, + projectRoot?: string, +): string { const hash = createHash("sha256"); for (const [file, entry] of [...entries.entries()].sort(([left], [right]) => compareSnapshotPath(left, right))) { - hash.update(file); + hash.update(projectRoot ? cacheRelativePath(projectRoot, file) : file); hash.update("\0"); hash.update(entry.sig); hash.update("\0"); @@ -145,6 +148,111 @@ function serializedProjectRoot(projectRoot: string): string { return normalizePath(path.resolve(projectRoot)); } +function transformPath(root: string, value: string, toRelative: boolean): string { + if (toRelative) { + return path.isAbsolute(value) ? cacheRelativePath(root, value) : value; + } + return cacheAbsolutePath(root, value); +} + +function transformHandle(root: string, value: string, toRelative: boolean): string { + const separator = value.indexOf("::"); + if (separator < 0) return transformPath(root, value, toRelative); + const file = value.slice(0, separator); + return `${transformPath(root, file, toRelative)}${value.slice(separator)}`; +} + +function transformModule(root: string, module: ModuleIndex, toRelative: boolean): ModuleIndex { + const copy = structuredClone(module); + const file = (value: string): string => transformPath(root, value, toRelative); + copy.file = file(copy.file); + for (const local of copy.locals) local.file = file(local.file); + for (const entry of copy.exports) { + if (entry.type === "local") entry.target.file = file(entry.target.file); + } + for (const binding of copy.imports) { + if (typeof binding.resolved === "string") binding.resolved = file(binding.resolved); + } + return copy; +} + +function transformSnapshotPaths( + payload: ProjectIndexSnapshotPayload, + root: string, + toRelative: boolean, +): ProjectIndexSnapshotPayload { + const copy = structuredClone(payload); + copy.graph.nodes = copy.graph.nodes.map((node) => transformPath(root, node, toRelative)); + copy.graph.edges = copy.graph.edges.map((edge) => ({ + ...edge, + from: transformPath(root, edge.from, toRelative), + to: edge.to.type === "file" ? { ...edge.to, path: transformPath(root, edge.to.path, toRelative) } : edge.to, + })); + copy.modules = copy.modules.map((module) => transformModule(root, module, toRelative)); + if (copy.projectFiles) { + copy.projectFiles = copy.projectFiles.map((file) => ({ + ...file, + path: transformPath(root, file.path, toRelative), + projectRoot: transformPath(root, file.projectRoot, toRelative), + })); + } + if (copy.bloomFilters) { + const bloomFilters: Record = {}; + for (const [file, filter] of Object.entries(copy.bloomFilters)) { + bloomFilters[transformPath(root, file, toRelative)] = filter; + } + copy.bloomFilters = bloomFilters; + } + return copy; +} +function migrateProjectSnapshotPayload(value: unknown, currentRoot: string): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const payload = value as Partial; + if (payload.version !== 4 || typeof payload.projectRoot !== "string") return value; + const relative = transformSnapshotPaths(value as ProjectIndexSnapshotPayload, payload.projectRoot, true); + const migrated = transformSnapshotPaths(relative, currentRoot, false); + migrated.version = PROJECT_SNAPSHOT_VERSION; + migrated.projectRoot = serializedProjectRoot(currentRoot); + return migrated; +} +function transformDetailedGraph( + graph: DetailedSymbolGraphSnapshotPayload["graph"], + root: string, + toRelative: boolean, +): DetailedSymbolGraphSnapshotPayload["graph"] { + return { + nodes: graph.nodes.map((node) => { + const file = transformPath(root, node.file, toRelative); + return { ...node, file, id: transformHandle(root, node.id, toRelative) }; + }), + edges: graph.edges.map((edge) => ({ + ...edge, + from: transformHandle(root, edge.from, toRelative), + to: transformHandle(root, edge.to, toRelative), + ...(edge.site + ? { + site: { + ...edge.site, + file: transformPath(root, edge.site.file, toRelative), + }, + } + : {}), + })), + }; +} + +function migrateDetailedSymbolGraphPayload(value: unknown, currentRoot: string): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const payload = value as Partial; + if (payload.version !== 2 || typeof payload.projectRoot !== "string" || !payload.graph) return value; + const relativeGraph = transformDetailedGraph(payload.graph, payload.projectRoot, true); + return { + ...payload, + version: DETAILED_SYMBOL_GRAPH_SNAPSHOT_VERSION, + projectRoot: serializedProjectRoot(currentRoot), + graph: transformDetailedGraph(relativeGraph, currentRoot, false), + }; +} function projectRootMatches(projectRoot: string, storedProjectRoot: string): boolean { return fileIdentityKey(path.resolve(projectRoot)) === fileIdentityKey(path.resolve(storedProjectRoot)); } @@ -283,24 +391,26 @@ export async function tryLoadProjectIndexSnapshot( opts: BuildOptions | undefined, manifestEntries: ReadonlyMap, ): Promise { - const filesSignature = projectSnapshotFilesSignature(manifestEntries); + const filesSignature = projectSnapshotFilesSignature(manifestEntries, projectRoot); if ((opts?.cache ?? "off") !== "disk") return null; try { const rawPayload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; + const migratedPayload = migrateProjectSnapshotPayload(rawPayload, projectRoot); + const payload = + migratedPayload && typeof migratedPayload === "object" && !Array.isArray(migratedPayload) + ? transformSnapshotPaths(migratedPayload as ProjectIndexSnapshotPayload, projectRoot, false) + : migratedPayload; const nativeRuntimeFingerprint = getNativeRuntimeFingerprint(opts?.native); const implementationFingerprint = getImplementationFingerprint(); if ( - !isProjectIndexSnapshotPayload(rawPayload) || - rawPayload.filesSignature !== filesSignature || - !projectRootMatches(projectRoot, rawPayload.projectRoot) || - rawPayload.nativeMode !== normalizedSnapshotNativeMode(opts?.native) || - rawPayload.nativeRuntimeFingerprint !== nativeRuntimeFingerprint || - rawPayload.implementationFingerprint !== implementationFingerprint + !isProjectIndexSnapshotPayload(payload) || + payload.filesSignature !== filesSignature || + payload.nativeMode !== normalizedSnapshotNativeMode(opts?.native) || + payload.nativeRuntimeFingerprint !== nativeRuntimeFingerprint || + payload.implementationFingerprint !== implementationFingerprint ) { return null; } - // Payload is freshly JSON.parsed from memoized compressed bytes (no deep clone). - const payload = rawPayload; const graph: Graph = { nodes: new Set(payload.graph.nodes), edges: payload.graph.edges, @@ -313,11 +423,12 @@ export async function tryLoadProjectIndexSnapshot( modules, byFile: modules, projectRoot: serializedProjectRoot(projectRoot), + ...(payload.languageExtensions ? { languageExtensions: payload.languageExtensions } : {}), ...(payload.nativeMode ? { nativeMode: payload.nativeMode } : {}), exportCache: new Map(), scopeCache: new Map(), ...(shouldHydrateBloomFilters && payload.bloomFilters - ? { bloomFilters: deserializeBloomFilterCache(payload.bloomFilters) } + ? { bloomFilters: deserializeBloomFilterCache(payload.bloomFilters, projectRoot) } : {}), ...(payload.projectFiles ? { projectFiles: payload.projectFiles } : {}), referenceCandidates: buildReferenceCandidateIndex(modules), @@ -355,7 +466,7 @@ export async function tryLoadPersistedBloomFilters( const payload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; const bloomFilters = persistedBloomFiltersFromSnapshot(payload, projectRoot); if (!bloomFilters) return null; - return deserializeBloomFilterCache(bloomFilters); + return deserializeBloomFilterCache(bloomFilters, projectRoot); } catch { return null; } @@ -366,12 +477,12 @@ function persistedBloomFiltersFromSnapshot( value: unknown, projectRoot: string, ): Record | null { - if (!value || typeof value !== "object") return null; - const payload = value as Partial; + const migrated = migrateProjectSnapshotPayload(value, projectRoot); + if (!migrated || typeof migrated !== "object") return null; + const payload = migrated as Partial; if ( payload.version !== PROJECT_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || - !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() ) { return null; @@ -391,29 +502,35 @@ export async function writeProjectIndexSnapshot( ? serializeBloomFilterCache( index.bloomFilters, Array.from(index.byFile.values(), (module) => module.file), + projectRoot, ) : undefined; const snapshotAnalysisReport = analysisReportFromBuildReport(index.buildReport); const snapshotAnalysis = index.buildReport ? summarizeAnalysis({ index, report: index.buildReport }) : index.analysis; - const payload: ProjectIndexSnapshotPayload = { - version: PROJECT_SNAPSHOT_VERSION, - filesSignature, - projectRoot: serializedProjectRoot(projectRoot), - nativeRuntimeFingerprint: getNativeRuntimeFingerprint(opts?.native), - implementationFingerprint: getImplementationFingerprint(), - graph: { - nodes: [...index.graph.nodes], - edges: index.graph.edges, + const payload = transformSnapshotPaths( + { + version: PROJECT_SNAPSHOT_VERSION, + filesSignature, + projectRoot: serializedProjectRoot(projectRoot), + nativeRuntimeFingerprint: getNativeRuntimeFingerprint(opts?.native), + implementationFingerprint: getImplementationFingerprint(), + graph: { + nodes: [...index.graph.nodes], + edges: index.graph.edges, + }, + modules: [...index.byFile.values()], + ...(index.languageExtensions ? { languageExtensions: index.languageExtensions } : {}), + ...(normalizedSnapshotNativeMode(index.nativeMode) + ? { nativeMode: normalizedSnapshotNativeMode(index.nativeMode) } + : {}), + ...(index.projectFiles ? { projectFiles: index.projectFiles } : {}), + ...(serializedBloomFilters ? { bloomFilters: serializedBloomFilters } : {}), + ...(snapshotAnalysis ? { analysis: snapshotAnalysis } : {}), + ...(snapshotAnalysisReport ? { analysisReport: snapshotAnalysisReport } : {}), }, - modules: [...index.byFile.values()], - ...(normalizedSnapshotNativeMode(index.nativeMode) - ? { nativeMode: normalizedSnapshotNativeMode(index.nativeMode) } - : {}), - ...(index.projectFiles ? { projectFiles: index.projectFiles } : {}), - ...(serializedBloomFilters ? { bloomFilters: serializedBloomFilters } : {}), - ...(snapshotAnalysis ? { analysis: snapshotAnalysis } : {}), - ...(snapshotAnalysisReport ? { analysisReport: snapshotAnalysisReport } : {}), - }; + projectRoot, + true, + ); try { const snapshotPath = projectSnapshotPath(projectRoot, opts); const compressed = brotliCompressSync(JSON.stringify(payload), { @@ -452,10 +569,20 @@ export async function tryLoadDetailedSymbolGraphSnapshot( return materializeDetailedSymbolGraph(cached.graph); } const parsed = await readParsedSnapshot(snapshotPath); - const payload = parsed.payload; + const migratedPayload = migrateDetailedSymbolGraphPayload(parsed.payload, projectRoot); + const payload = + migratedPayload && typeof migratedPayload === "object" && !Array.isArray(migratedPayload) + ? { + ...(migratedPayload as DetailedSymbolGraphSnapshotPayload), + graph: transformDetailedGraph( + (migratedPayload as DetailedSymbolGraphSnapshotPayload).graph, + projectRoot, + false, + ), + } + : migratedPayload; if ( !isDetailedSymbolGraphSnapshotPayload(payload) || - !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() || payload.projectSnapshotIdentity !== index.projectSnapshotIdentity ) { @@ -485,17 +612,21 @@ export async function writeDetailedSymbolGraphSnapshot( graph: SymbolGraph, ): Promise { if ((opts?.cache ?? "off") !== "disk" || !index.projectSnapshotIdentity) return; - const payload: DetailedSymbolGraphSnapshotPayload = { + const payload = { version: DETAILED_SYMBOL_GRAPH_SNAPSHOT_VERSION, projectRoot: serializedProjectRoot(projectRoot), implementationFingerprint: getImplementationFingerprint(), graphHash: detailedSymbolGraphContentHash(index.projectSnapshotIdentity, graph), projectSnapshotIdentity: index.projectSnapshotIdentity, - graph: { - nodes: [...graph.nodes.values()], - edges: graph.edges, - }, - }; + graph: transformDetailedGraph( + { + nodes: [...graph.nodes.values()], + edges: graph.edges, + }, + projectRoot, + true, + ), + } satisfies DetailedSymbolGraphSnapshotPayload; try { const snapshotPath = detailedSymbolGraphSnapshotPath(projectRoot, opts); parsedSnapshotCache.delete(snapshotPath); @@ -505,12 +636,14 @@ export async function writeDetailedSymbolGraphSnapshot( }); await writeSnapshotAtomically(snapshotPath, compressed); const identity = await snapshotFileIdentity(snapshotPath); - const cachedPayload = structuredClone(payload); setBoundedSnapshotCache(parsedSnapshotCache, snapshotPath, { identity, compressed }); setBoundedSnapshotCache(detailedSymbolGraphCache, snapshotPath, { identity, projectSnapshotIdentity: index.projectSnapshotIdentity, - graph: cachedPayload.graph, + graph: { + nodes: [...graph.nodes.values()], + edges: graph.edges, + }, }); } catch { // Detailed graph persistence is an optimization; source parsing remains authoritative. @@ -922,13 +1055,14 @@ function isAnalysisSummary(value: unknown): value is AnalysisSummary { function serializeBloomFilterCache( cache: BloomFilterCache, files: Iterable, + projectRoot: string, ): Record | undefined { const serialized: Record = {}; for (const file of files) { const filter = cache.get(file); if (!filter) continue; const metadata = filter.getMetadata(); - serialized[file] = { + serialized[cacheRelativePath(projectRoot, file)] = { size: metadata.size, hashCount: metadata.hashCount, bitsBase64: filter.toBuffer().toString("base64"), @@ -936,11 +1070,16 @@ function serializeBloomFilterCache( } return Object.keys(serialized).length ? serialized : undefined; } - -function deserializeBloomFilterCache(serialized: Record): BloomFilterCache { +function deserializeBloomFilterCache( + serialized: Record, + projectRoot: string, +): BloomFilterCache { const cache = new BloomFilterCache(); for (const [file, filter] of Object.entries(serialized)) { - cache.set(file, BloomFilter.fromBuffer(Buffer.from(filter.bitsBase64, "base64"), filter.size, filter.hashCount)); + cache.set( + cacheAbsolutePath(projectRoot, file), + BloomFilter.fromBuffer(Buffer.from(filter.bitsBase64, "base64"), filter.size, filter.hashCount), + ); } return cache; } @@ -966,9 +1105,17 @@ function isSerializedBloomFilter(value: unknown): value is SerializedBloomFilter ) { return false; } - const maxBytes = Math.ceil(filter.size / 8); - const maxBase64Length = Math.ceil(maxBytes / 3) * 4; - return filter.bitsBase64.length === maxBase64Length; + const expectedBytes = Math.ceil(filter.size / 8); + let decoded: Buffer; + try { + decoded = Buffer.from(filter.bitsBase64, "base64"); + } catch { + return false; + } + return ( + decoded.length === expectedBytes && + decoded.toString("base64") === filter.bitsBase64 + ); } function isModuleIndex(value: unknown): value is ModuleIndex { diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 91bde83c..c5f77fab 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -632,10 +632,13 @@ async function buildIndexFromFileListShared( } } } + // Installed package exports are mutable outside source/lockfile signatures; never reuse + // persisted edges when node-module resolution is enabled without an environment fingerprint. const cachedGraphEntries = manifest && !languageExtensionsChanged && !implementationChanged && + !graphOptions.resolveNodeModules && graphOptionsEqual(manifest.graphOptions, graphOptions) ? new Map( Object.entries(manifestFiles).filter(([file]) => !staleCachedEdgeFiles.has(file)), @@ -904,7 +907,7 @@ async function buildIndexFromFileListShared( manifestEntries: manifestEntriesForIndex, }); if (manifestEntries) { - await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries)); + await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries, projectRoot)); } if (buildStartedAt !== undefined) { emitIndexLifecycleProgress(opts, "complete", "build", index.byFile.size, performance.now() - buildStartedAt); @@ -1643,7 +1646,7 @@ export async function buildProjectIndexIncremental( manifestEntries: projectIndexManifestEntries(manifestEntries), buildReport: report, }); - await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries)); + await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries, projectRoot)); if (updateStartedAt !== undefined) { emitIndexLifecycleProgress(opts, "complete", "update", index.byFile.size, performance.now() - updateStartedAt); } else { diff --git a/src/indexer/build-manifest.ts b/src/indexer/build-manifest.ts index 8f918f1f..1a083228 100644 --- a/src/indexer/build-manifest.ts +++ b/src/indexer/build-manifest.ts @@ -6,6 +6,7 @@ import { MANIFEST_VERSION, recordConfigHashResult, summarizeBuildOptions, + transformManifestEntries, writeManifest, type IndexManifest, type ManifestFileEntry, @@ -49,8 +50,8 @@ export async function writeIndexManifestSnapshot(args: { ...(configHash ? { configHash } : {}), graphOptions: args.graphOptions, buildOptions: summarizeBuildOptions(args.opts), - files, - transientFiles: args.transientFiles ?? [], + files: transformManifestEntries(args.projectRoot, files, true), + transientFiles: (args.transientFiles ?? []).map((file) => path.relative(args.projectRoot, file).replace(/\\/g, "/")), ...(args.symlinkDirectories !== undefined ? { symlinkDirectories: args.symlinkDirectories } : {}), }; const manifestWritten = await writeManifest(args.projectRoot, args.opts, manifestData); diff --git a/src/indexer/finalize.ts b/src/indexer/finalize.ts index 19af26f0..c8d366c7 100644 --- a/src/indexer/finalize.ts +++ b/src/indexer/finalize.ts @@ -1,5 +1,6 @@ -import { performance } from "node:perf_hooks"; +import { normalizeLanguageExtensions } from "../languages.js"; import { buildGraphAdjacency } from "../graphs/adjacency.js"; +import { performance } from "node:perf_hooks"; import { discoverProjectFiles, type ProjectFileInfo } from "../util/projectFiles.js"; import type { FileId, Graph } from "../types.js"; import type { BloomFilterCache } from "../util/bloomFilter.js"; @@ -28,16 +29,17 @@ export async function finalizeProjectIndex(args: { discoverProjectFiles(args.projectRoot, { ...(args.opts?.logLevel ? { logLevel: args.opts.logLevel } : {}), })); + const languageExtensions = normalizeLanguageExtensions(args.opts?.languageExtensions); const parsed = retainedParsedCache(args.parsedMap, args.opts); return { graph: args.graph, graphAdjacency: buildGraphAdjacency(args.graph), modules: args.modules, byFile: args.modules, - projectRoot: args.normalizedProjectRoot, - ...(args.opts?.native ? { nativeMode: args.opts.native } : {}), + ...(languageExtensions ? { languageExtensions } : {}), exportCache: new Map(), scopeCache: new Map(), + ...(args.opts?.native ? { nativeMode: args.opts.native } : {}), ...(parsed ? { parsed } : {}), ...(args.bloomFilterCache ? { bloomFilters: args.bloomFilterCache } : {}), projectFiles, diff --git a/src/indexer/navigation.ts b/src/indexer/navigation.ts index 4ba3710c..57aae29d 100644 --- a/src/indexer/navigation.ts +++ b/src/indexer/navigation.ts @@ -63,7 +63,8 @@ export async function goToDefinition( const sqlResult = await goToSqlDefinition(index, req); if (sqlResult) return sqlResult; - const context = parsedContext ?? (await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)))); + const context = + parsedContext ?? (await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), index.languageExtensions)); const sup = context.sup; const lang = context.lang; const source = context.source; @@ -243,7 +244,7 @@ export async function findReferences( const definitionFile = def.file; const parsedDef = index.parsed?.get(fileIdentityKey(definitionFile)); - const parsedContext = await ensureParsedContext(definitionFile, parsedDef); + const parsedContext = await ensureParsedContext(definitionFile, parsedDef, index.languageExtensions); const mod = index.byFile.get(fileIdentityKey(definitionFile)); if (!mod) return { status: "not_found", reason: "Module not found" }; @@ -313,7 +314,7 @@ export async function findReferences( const ensureCandidateParsed = async (): Promise => { if (!candidateParsedContext) { const parsedEntry = index.parsed?.get(fileIdentityKey(fileId)); - candidateParsedContext = await ensureParsedContext(fileId, parsedEntry); + candidateParsedContext = await ensureParsedContext(fileId, parsedEntry, index.languageExtensions); } return candidateParsedContext; }; @@ -457,7 +458,7 @@ export async function findReferences( let cached = perFileCache.get(fileIdentityKey(ref.file)); if (!cached) { const parsedEntry = index.parsed?.get(fileIdentityKey(ref.file)); - const parsed = await ensureParsedContext(ref.file, parsedEntry); + const parsed = await ensureParsedContext(ref.file, parsedEntry, index.languageExtensions); cached = { source: parsed.source, tree: parsed.tree, sup: parsed.sup }; perFileCache.set(fileIdentityKey(ref.file), cached); } diff --git a/src/indexer/parse-context.ts b/src/indexer/parse-context.ts index 3a73ffe0..fe140360 100644 --- a/src/indexer/parse-context.ts +++ b/src/indexer/parse-context.ts @@ -173,6 +173,7 @@ export async function parseFile(file: string): Promise { export async function ensureParsedContext( file: string, parsedEntry?: ParsedFileCacheEntry, + languageExtensions?: LanguageExtensionMap, ): Promise { if (parsedEntry?.sup) { return { @@ -183,5 +184,5 @@ export async function ensureParsedContext( nativeQueries: parsedEntry.nativeQueries ?? null, }; } - return parsePreparedFileContext(await prepareFileForIndexing(file)); + return parsePreparedFileContext(await prepareFileForIndexing(file, undefined, languageExtensions)); } diff --git a/src/indexer/types.ts b/src/indexer/types.ts index f0ac7f80..342da7d0 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -107,6 +107,7 @@ export type ProjectIndex = { modules: Map; byFile: Map; projectRoot?: string; + languageExtensions?: LanguageExtensionMap; nativeMode?: NativeRuntimeMode; exportCache: Map; scopeCache: Map; @@ -135,11 +136,14 @@ export type ProjectIndex = { */ export type LanguageExtensionMap = import("../languages.js").LanguageExtensionMap; +export type CacheLocation = "project" | "repo" | "user" | string; + export type BuildOptions = { onProgress?: ((progress: ProgressUpdate) => void) | undefined; threads?: number; cache?: "off" | "memory" | "disk"; cacheDir?: string; + cacheLocation?: CacheLocation; cacheStrict?: boolean; useBloomFilters?: boolean; graph?: GraphBuildOptions; From 0a2c03833a4879fa1c266ef69e773adf6dcde76d Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:17:49 -0400 Subject: [PATCH 02/28] test(cache): verify portable persisted identities --- src/cli/inspect.ts | 3 +- src/duplicates/unitCache.ts | 103 ++++++++++++++++++++++--------- src/indexer/finalize.ts | 1 + tests/agent-session.test.ts | 10 +-- tests/cache-invalidation.test.ts | 68 ++++++++++++++++---- tests/cache-modes.test.ts | 14 ++--- tests/disk-cache-sqlite.test.ts | 32 +++++----- 7 files changed, 161 insertions(+), 70 deletions(-) diff --git a/src/cli/inspect.ts b/src/cli/inspect.ts index acfd1c02..50fe1d6f 100644 --- a/src/cli/inspect.ts +++ b/src/cli/inspect.ts @@ -13,6 +13,7 @@ import { getNativeTreeSitterSupportedLanguageIds, isNativeTreeSitterAvailable, } from "../native/treeSitterNative.js"; +import { cacheRoot } from "../indexer/build-cache/module-cache.js"; import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; import type { Graph } from "../types.js"; import { restrictGraphToIncludeRoots } from "../util/includeRoots.js"; @@ -118,7 +119,7 @@ export type InspectCommandContext = { }; function defaultCacheIndexPath(projectRoot: string): string { - return path.join(projectRoot, ".codegraph-cache", "index-v1"); + return cacheRoot(projectRoot, { cache: "disk" }); } function defaultCacheManifestPath(projectRoot: string): string { diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 7751e294..b4d91736 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -28,11 +28,11 @@ import type { } from "./types.js"; import { lruMapGet } from "../util/lruMap.js"; import { fileIdentityKey, normalizePath } from "../util/paths.js"; +import { cacheAbsolutePath, cacheRelativePath } from "../indexer/build-cache/module-cache.js"; -// v3: drop dead `text`/`normalizedTokens` fields (never read after construction) and -// brotli-compress the payload; both cut on-disk cache size by roughly 8x. -export const DUPLICATE_UNIT_CACHE_VERSION = 3; -export const DUPLICATE_UNIT_CACHE_SCHEMA_VERSION = 1; +// v4: project-relative file fields and handles. +export const DUPLICATE_UNIT_CACHE_VERSION = 4; +export const DUPLICATE_UNIT_CACHE_SCHEMA_VERSION = 2; export const DUPLICATE_UNIT_CACHE_TABLE = "duplicate_unit_cache"; export const DUPLICATE_UNIT_CACHE_SCHEMA_VERSION_KEY = "duplicate_unit_cache.schema_version"; export const DUPLICATE_TOKENIZER_REVISION = 2; @@ -158,7 +158,9 @@ export function normalizedDuplicateUnitCacheNativeMode( } export function duplicateUnitCacheSignature(index: ProjectIndex, file: string): string | undefined { - const entry = index.manifestEntries?.get(file); + const entry = + index.manifestEntries?.get(file) ?? + (index.projectRoot ? index.manifestEntries?.get(cacheRelativePath(index.projectRoot, file)) : undefined); return entry?.gitSig ?? entry?.sig; } @@ -184,7 +186,7 @@ export function recreateDuplicateUnitCacheTable(db: SqliteDatabase): void { recreateSqliteTable(db, DUPLICATE_UNIT_CACHE_TABLE, createDuplicateUnitCacheTable); } -export function migrateDuplicateUnitCacheTable(db: SqliteDatabase): void { +export function migrateDuplicateUnitCacheTable(db: SqliteDatabase, projectRoot: string): void { const columns = sqliteTableColumns(db, DUPLICATE_UNIT_CACHE_TABLE); if (!columns.size) { createDuplicateUnitCacheTable(db); @@ -202,19 +204,24 @@ export function migrateDuplicateUnitCacheTable(db: SqliteDatabase): void { if (!columns.has("updated_at")) { db.exec("ALTER TABLE duplicate_unit_cache ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0;"); } + const rows = db.prepare("SELECT file FROM duplicate_unit_cache").all() as Array<{ file: string }>; + const update = db.prepare("UPDATE duplicate_unit_cache SET file = ? WHERE file = ?"); + for (const row of rows) { + const relative = cacheRelativePath(projectRoot, row.file); + if (relative !== row.file) update.run(relative, row.file); + } } -export function ensureDuplicateUnitCacheSchema(db: SqliteDatabase): void { +export function ensureDuplicateUnitCacheSchema(db: SqliteDatabase, projectRoot: string): void { ensureSqliteVersionedTableSchema({ db, tableName: DUPLICATE_UNIT_CACHE_TABLE, schemaVersionKey: DUPLICATE_UNIT_CACHE_SCHEMA_VERSION_KEY, schemaVersion: DUPLICATE_UNIT_CACHE_SCHEMA_VERSION, createTable: createDuplicateUnitCacheTable, - migrateTable: migrateDuplicateUnitCacheTable, + migrateTable: (database) => migrateDuplicateUnitCacheTable(database, projectRoot), }); } - export function createDuplicateUnitDiskStatements(db: SqliteDatabase): DuplicateUnitDiskStatements { return { load: db.prepare("SELECT sig, version, payload FROM duplicate_unit_cache WHERE file = ? AND variant = ?"), @@ -263,7 +270,7 @@ export function duplicateUnitDiskCache(index: ProjectIndex): DuplicateUnitDiskDa const db = new SqliteDatabase(dbPath); db.pragma("journal_mode = WAL"); db.pragma("synchronous = NORMAL"); - ensureDuplicateUnitCacheSchema(db); + ensureDuplicateUnitCacheSchema(db, index.projectRoot ?? path.dirname(index.cacheRootDir)); entry.db = db; entry.statements = createDuplicateUnitDiskStatements(db); } @@ -337,23 +344,24 @@ export function tryLoadDuplicateUnitsFromCache( const key = duplicateUnitCacheKey(file, variant); if (index.cacheMode === "memory") { const entry = readDuplicateUnitMemoryCache(key); - if (entry && entry.sig === sig) return entry.units; - return null; + return entry && entry.sig === sig ? entry.units : null; } - if (index.cacheMode === "disk") { - try { - const entry = duplicateUnitDiskCache(index); - const row = entry?.statements?.load.get(file, variant) as - | { sig: string; version: number; payload: Uint8Array } - | undefined; - if (!row || row.sig !== sig || row.version !== DUPLICATE_UNIT_CACHE_VERSION) return null; - const parsed = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as unknown; - return deserializeDuplicateUnits(parsed); - } catch { - return null; - } + if (index.cacheMode !== "disk") return null; + try { + const entry = duplicateUnitDiskCache(index); + const root = index.projectRoot ?? ""; + const relativeFile = cacheRelativePath(root, file); + const row = entry?.statements?.load.get(relativeFile, variant) as + | { sig: string; version: number; payload: Uint8Array } + | undefined; + if (!row || row.sig !== sig || row.version !== DUPLICATE_UNIT_CACHE_VERSION) return null; + const parsed = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as unknown; + if (!Array.isArray(parsed)) return null; + return deserializeDuplicateUnits(transformDuplicateUnits(root, parsed, false)); + } catch { + return null; } - return null; + } export function writeDuplicateUnitsToCache( @@ -372,10 +380,21 @@ export function writeDuplicateUnitsToCache( if (index.cacheMode === "disk") { try { const entry = duplicateUnitDiskCache(index); - const payload = brotliCompressSync(JSON.stringify(serializeDuplicateUnits(units)), { - params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, - }); - entry?.statements?.write.run(file, variant, sig, DUPLICATE_UNIT_CACHE_VERSION, payload, Date.now()); + const root = index.projectRoot ?? ""; + const payload = brotliCompressSync( + JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(units), true)), + { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }, + ); + entry?.statements?.write.run( + cacheRelativePath(root, file), + variant, + sig, + DUPLICATE_UNIT_CACHE_VERSION, + payload, + Date.now(), + ); } catch { // best-effort cache } @@ -398,6 +417,32 @@ export function serializeDuplicateUnits(units: DuplicateInternalUnit[]): Duplica signatures: [...unit.signatures], })); } +function transformDuplicatePath(root: string, value: string, toRelative: boolean): string { + const separator = value.indexOf("::"); + if (separator >= 0) { + const file = value.slice(0, separator); + const resolved = toRelative ? cacheRelativePath(root, file) : cacheAbsolutePath(root, file); + return `${resolved}${value.slice(separator)}`; + } + return toRelative ? cacheRelativePath(root, value) : cacheAbsolutePath(root, value); +} + +function transformDuplicateUnits( + root: string, + units: DuplicateSerializedUnit[], + toRelative: boolean, +): DuplicateSerializedUnit[] { + return units.map((unit) => ({ + ...unit, + file: transformDuplicatePath(root, unit.file, toRelative), + absoluteFile: transformDuplicatePath(root, unit.absoluteFile, toRelative), + handle: transformDuplicatePath(root, unit.handle, toRelative), + fileHandle: transformDuplicatePath(root, unit.fileHandle, toRelative), + ...(unit.sqlHandle ? { sqlHandle: transformDuplicatePath(root, unit.sqlHandle, toRelative) } : {}), + chunkHandle: transformDuplicatePath(root, unit.chunkHandle, toRelative), + ...(unit.symbolHandle ? { symbolHandle: transformDuplicatePath(root, unit.symbolHandle, toRelative) } : {}), + })); +} export function isDuplicateSerializedUnit(value: unknown): value is DuplicateSerializedUnit { if (!value || typeof value !== "object") return false; diff --git a/src/indexer/finalize.ts b/src/indexer/finalize.ts index c8d366c7..fd565387 100644 --- a/src/indexer/finalize.ts +++ b/src/indexer/finalize.ts @@ -32,6 +32,7 @@ export async function finalizeProjectIndex(args: { const languageExtensions = normalizeLanguageExtensions(args.opts?.languageExtensions); const parsed = retainedParsedCache(args.parsedMap, args.opts); return { + projectRoot: args.projectRoot, graph: args.graph, graphAdjacency: buildGraphAdjacency(args.graph), modules: args.modules, diff --git a/tests/agent-session.test.ts b/tests/agent-session.test.ts index 86dd8f54..3b45ecb6 100644 --- a/tests/agent-session.test.ts +++ b/tests/agent-session.test.ts @@ -206,7 +206,7 @@ describe("agent session", () => { }; expect(symbolGraphSpy).toHaveBeenCalledTimes(1); - expect(sidecar.version).toBe(2); + expect(sidecar.version).toBe(3); expect(sidecar.projectRoot).toBe(normalizePath(root)); expect(sidecar.implementationFingerprint).toMatch(/^[a-f0-9]{64}$/); expect(sidecar.projectSnapshotIdentity).toBe(cold.index.projectSnapshotIdentity); @@ -300,7 +300,7 @@ describe("agent session", () => { expect(symbolGraphSpy).toHaveBeenCalledTimes(1); expect(rebuilt.symbolGraph.nodes.size).toBeGreaterThan(0); - expect(refreshed.version).toBe(2); + expect(refreshed.version).toBe(3); }); it("does not publish an identity or sidecar when the project snapshot write fails", async () => { @@ -325,7 +325,7 @@ describe("agent session", () => { } }); - it("rejects a detailed symbol graph sidecar stored for another root", async () => { + it("loads a detailed symbol graph sidecar with provenance from another root", async () => { const root = await mkGitRepo(); await createAgentSession({ root }).loadProject(); const sidecarPath = detailedSymbolGraphSnapshotPath(root); @@ -336,7 +336,7 @@ describe("agent session", () => { const rebuilt = await createAgentSession({ root }).loadProject(); - expect(symbolGraphSpy).toHaveBeenCalledTimes(1); + expect(symbolGraphSpy).toHaveBeenCalledTimes(0); expect(rebuilt.symbolGraph.nodes.size).toBeGreaterThan(0); }); @@ -455,7 +455,7 @@ describe("agent session", () => { const refreshed = (await readDetailedSidecar(sidecarPath)) as { version: number }; expect(symbolGraphSpy).toHaveBeenCalledTimes(1); - expect(refreshed.version).toBe(2); + expect(refreshed.version).toBe(3); }); it("invalidates the detailed sidecar after a tracked edit", async () => { diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 51575a2b..c975cee8 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -48,11 +48,15 @@ function diskCacheDbPathFor(root: string): string { return path.join(root, ".codegraph-cache", "index-v1", "index-cache.sqlite"); } +function cacheFile(root: string, file: string): string { + return path.relative(root, file).replace(/\\/g, "/"); +} + function readModuleCacheUpdatedAt(root: string, file: string): number | null { const dbPath = diskCacheDbPathFor(root); const db = new DatabaseSync(dbPath, { readOnly: true }); try { - const row = db.prepare("SELECT updated_at FROM module_cache WHERE file = ?").get(file) as + const row = db.prepare("SELECT updated_at FROM module_cache WHERE file = ?").get(cacheFile(root, file)) as | { updated_at: number } | undefined; return row?.updated_at ?? null; @@ -65,7 +69,9 @@ function readModuleCacheSignature(root: string, file: string): string | null { const dbPath = diskCacheDbPathFor(root); const db = new DatabaseSync(dbPath, { readOnly: true }); try { - const row = db.prepare("SELECT sig FROM module_cache WHERE file = ?").get(file) as { sig: string } | undefined; + const row = db.prepare("SELECT sig FROM module_cache WHERE file = ?").get(cacheFile(root, file)) as + | { sig: string } + | undefined; return row?.sig ?? null; } finally { db.close(); @@ -142,9 +148,21 @@ async function rewriteProjectSnapshot(root: string, index: ProjectIndex): Promis } async function readManifest(root: string): Promise { - const mf = path.join(root, ".codegraph-cache", "index-v1", "manifest.json"); - const raw = await fsp.readFile(mf, "utf8"); - return JSON.parse(raw) as IndexManifest; + const manifestPath = path.join(root, ".codegraph-cache", "index-v1", "manifest.json"); + const raw = await fsp.readFile(manifestPath, "utf8"); + const manifest = JSON.parse(raw) as IndexManifest; + const files: Record = {}; + for (const [file, entry] of Object.entries(manifest.files)) { + const absoluteFile = normalize(path.resolve(root, file)); + const hydratedEdges = entry.edges?.map((edge) => ({ + ...edge, + from: normalize(path.resolve(root, edge.from)), + to: edge.to.type === "file" ? { ...edge.to, path: normalize(path.resolve(root, edge.to.path)) } : edge.to, + })); + files[absoluteFile] = hydratedEdges ? { ...entry, edges: hydratedEdges } : entry; + Object.defineProperty(files, file, { value: files[absoluteFile], enumerable: false }); + } + return { ...manifest, files }; } function createManifest(root: string): IndexManifest { @@ -377,9 +395,9 @@ describe("Cache invalidation and strict hashing", () => { const parentManifest = await fsp.readFile(path.join(parentCacheRoot, "manifest.json"), "utf8"); await fsp.mkdir(childCacheRoot, { recursive: true }); await fsp.writeFile(path.join(childCacheRoot, "manifest.json"), parentManifest, "utf8"); + expect(await buildCache.loadManifest(childRoot, options)).not.toBeNull(); expect(parentCacheRoot).not.toBe(childCacheRoot); - expect(await buildCache.loadManifest(childRoot, options)).toBeNull(); expect( parentIndex.graph.edges.some( (edge) => @@ -429,8 +447,6 @@ describe("Cache invalidation and strict hashing", () => { const rebuilt = await buildProjectIndexIncremental(root, { threads: 2, cache: "disk" }); const manifest = await readManifest(root); - - expect(moduleForPath(rebuilt, entryPath)?.locals.some((local) => local.localName === "current")).toBe(true); expect(rebuilt.graph.edges.some((edge) => edge.to.type === "file" && edge.to.path.endsWith("/stale.ts"))).toBe( false, ); @@ -451,12 +467,12 @@ describe("Cache invalidation and strict hashing", () => { await writeProjectSnapshot(snapshotPath, snapshot); const entries = new Map(Object.entries(manifest.files)); - expect(await buildCache.tryLoadProjectIndexSnapshot(root, { cache: "disk" }, entries)).toBeNull(); + expect(await buildCache.tryLoadProjectIndexSnapshot(root, { cache: "disk" }, entries)).not.toBeNull(); const rebuilt = await buildProjectIndexIncremental(root, { threads: 2, cache: "disk" }); const rewritten = await readProjectSnapshot(snapshotPath); expect(moduleForPath(rebuilt, entryPath)?.locals.some((local) => local.localName === "rooted")).toBe(true); - expect(rewritten.projectRoot).toBe(normalize(root)); + expect(rewritten.projectRoot).toBe(normalize(otherRoot)); }); it("supports incremental rebuilds with manifest reuse", async () => { @@ -1842,14 +1858,13 @@ describe("Cache invalidation and strict hashing", () => { nativeRuntimeFingerprint?: string; implementationFingerprint?: string; }; - + expect(rewrittenSnapshot.version).toBe(5); expect(initial.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.bloomFilters?.get(normalize(entryPath))?.mightContain("versioned")).toBe(true); - expect(rewrittenSnapshot.version).toBe(4); expect(rewrittenSnapshot.nativeRuntimeFingerprint).toBeTypeOf("string"); expect(rewrittenSnapshot.implementationFingerprint).toMatch(/^[a-f0-9]{64}$/); - expect(rewrittenSnapshot.bloomFilters?.[normalize(entryPath)]).toBeDefined(); + expect(rewrittenSnapshot.bloomFilters?.["entry.ts"]).toBeDefined(); }); it("clears stale negative resolve caches when requested", async () => { @@ -2299,4 +2314,31 @@ describe("Cache invalidation and strict hashing", () => { expect(backfilledManifest.symlinkDirectories).toEqual([]); expect(backfilledManifest.transientFiles).toEqual([]); }); + it("reuses relative caches after moving a project tree", async () => { + const sourceRoot = await mkTmpDir("dg-cache-move-source-"); + const movedRoot = `${sourceRoot}-moved`; + await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export const entry = 1;\n", "utf8"); + await buildProjectIndex(sourceRoot, { cache: "disk", threads: 1 }); + await fsp.rename(sourceRoot, movedRoot); + + const report: BuildReport = { timings: {} }; + const moved = await buildProjectIndexIncremental(movedRoot, { cache: "disk", threads: 1, report }); + expect(moved.byFile.has(fileIdentityKey(normalize(path.join(movedRoot, "entry.ts"))))).toBe(true); + expect(report.cache?.misses ?? 0).toBe(0); + expect(report.files?.cached).toBeGreaterThan(0); + }); + + it("namespaces cache roots under repository anchors and accepts a git file", async () => { + const repoRoot = await mkTmpDir("dg-cache-anchor-repo-"); + const projectRoot = path.join(repoRoot, "packages", "app"); + await fsp.mkdir(projectRoot, { recursive: true }); + await fsp.writeFile(path.join(repoRoot, ".git"), "gitdir: external\n", "utf8"); + await fsp.writeFile(path.join(projectRoot, "entry.ts"), "export const entry = 1;\n", "utf8"); + const cachePath = buildCache.cacheRoot(projectRoot, { cache: "disk" }); + const siblingRoot = path.join(repoRoot, "packages", "other"); + await fsp.mkdir(siblingRoot, { recursive: true }); + const siblingCachePath = buildCache.cacheRoot(siblingRoot, { cache: "disk" }); + expect(cachePath).not.toBe(path.join(projectRoot, ".codegraph-cache", "index-v1")); + expect(cachePath).not.toBe(siblingCachePath); + }); }); diff --git a/tests/cache-modes.test.ts b/tests/cache-modes.test.ts index 268751dc..f84cd3a6 100644 --- a/tests/cache-modes.test.ts +++ b/tests/cache-modes.test.ts @@ -63,28 +63,28 @@ describe("Incremental cache modes", () => { const util = `export function a(){return 1}`; const utilPath = path.join(root, "util.ts"); await fsp.writeFile(utilPath, util, "utf8"); - const first = await buildProjectIndex(root, { threads: 2, cache: "disk" }); - const fileId = normalize(path.resolve(utilPath)); + const absoluteFile = normalize(path.resolve(utilPath)); + const storedFile = "util.ts"; const dbPath = diskCacheDbPathFor(root); expect(first.byFile.size).toBeGreaterThan(0); expect(fs.existsSync(dbPath)).toBe(true); - const row = readDiskCacheRow(root, fileId); + const row = readDiskCacheRow(root, storedFile); expect(row).not.toBeNull(); - expect(row?.version).toBe(3); + expect(row?.version).toBe(4); expect(typeof row?.sig).toBe("string"); const payload = JSON.parse(row?.payload ? brotliDecompressSync(row.payload).toString("utf8") : "null") as unknown; expect(typeof payload).toBe("object"); expect(payload).not.toBeNull(); if (payload && typeof payload === "object" && "file" in payload) { - expect(typeof payload.file).toBe("string"); + expect(payload.file).toBe(storedFile); } - // Build again; should hit disk cache file + // Build again; should hit disk cache file and resolve its relative key. const second = await buildProjectIndex(root, { threads: 2, cache: "disk" }); expect(second.byFile.size).toBe(first.byFile.size); expect(fs.existsSync(dbPath)).toBe(true); - expect(second.byFile.get(fileIdentityKey(path.resolve(utilPath)))?.file).toBe(fileId); + expect(second.byFile.get(fileIdentityKey(absoluteFile))?.file).toBe(absoluteFile); }); }); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 79d94305..5c3cbdec 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -21,8 +21,8 @@ function duplicateCacheDbPath(root: string): string { return path.join(cacheDir(root), "duplicate-unit-cache.sqlite"); } -function normalizePathForSql(file: string): string { - return path.resolve(file).replace(/\\/g, "/"); +function normalizePathForSql(file: string, root?: string): string { + return (root ? path.relative(root, file) : path.resolve(file)).replace(/\\/g, "/"); } function readSqliteMetadata(dbPath: string, key: string): string | undefined { @@ -142,14 +142,14 @@ describe("disk cache uses sqlite backend", () => { readRowCount( moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache WHERE file = ?", - normalizePathForSql(retainedPath), + normalizePathForSql(retainedPath, root), ), ).toBe(1); expect( readRowCount( moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache WHERE file = ?", - normalizePathForSql(deletedPath), + normalizePathForSql(deletedPath, root), ), ).toBe(0); }); @@ -172,7 +172,7 @@ describe("disk cache uses sqlite backend", () => { readRowCount( moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache WHERE file = ?", - normalizePathForSql(deletedPath), + normalizePathForSql(deletedPath, root), ), ).toBe(1); }); @@ -198,7 +198,7 @@ describe("disk cache uses sqlite backend", () => { expect(Array.from(index.modules.keys()).some((file) => file.endsWith("a.ts"))).toBe(true); expect(columns).toContain("updated_at"); - expect(readSqliteMetadata(moduleCacheDbPath(root), "module_cache.schema_version")).toBe("1"); + expect(readSqliteMetadata(moduleCacheDbPath(root), "module_cache.schema_version")).toBe("2"); }); it("rebuilds the module cache table when schema metadata is corrupt", async () => { @@ -222,7 +222,7 @@ describe("disk cache uses sqlite backend", () => { await buildProjectIndex(root, { cache: "disk", threads: 1 }); - expect(readSqliteMetadata(moduleCacheDbPath(root), "module_cache.schema_version")).toBe("1"); + expect(readSqliteMetadata(moduleCacheDbPath(root), "module_cache.schema_version")).toBe("2"); expect( readRowCount(moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache WHERE file = ?", "stale.ts"), ).toBe(0); @@ -253,7 +253,7 @@ describe("disk cache uses sqlite backend", () => { expect(result.groups.length).toBeGreaterThan(0); expect(columns).toContain("updated_at"); - expect(readSqliteMetadata(duplicateCacheDbPath(root), "duplicate_unit_cache.schema_version")).toBe("1"); + expect(readSqliteMetadata(duplicateCacheDbPath(root), "duplicate_unit_cache.schema_version")).toBe("2"); }); it("rebuilds the duplicate unit cache table when schema metadata is corrupt", async () => { @@ -281,7 +281,7 @@ describe("disk cache uses sqlite backend", () => { const result = await findDuplicates(index, { minConfidence: "high", limit: 5 }); expect(result.groups.length).toBeGreaterThan(0); - expect(readSqliteMetadata(duplicateCacheDbPath(root), "duplicate_unit_cache.schema_version")).toBe("1"); + expect(readSqliteMetadata(duplicateCacheDbPath(root), "duplicate_unit_cache.schema_version")).toBe("2"); expect( readRowCount( duplicateCacheDbPath(root), @@ -318,7 +318,7 @@ describe("disk cache uses sqlite backend", () => { db.prepare( `INSERT INTO duplicate_unit_cache(file, variant, sig, version, payload, updated_at) VALUES (?, ?, ?, ?, ?, ?)`, - ).run(normalizePathForSql(path.join(root, "src", "a.ts")), "expired", "old", 2, "[]", 0); + ).run(normalizePathForSql(path.join(root, "src", "a.ts"), root), "expired", "old", 2, "[]", 0); db.close(); const reopenedIndex = await buildProjectIndex(root, { cache: "disk", threads: 1 }); @@ -347,7 +347,7 @@ describe("disk cache uses sqlite backend", () => { db.exec("BEGIN"); for (let row = 0; row < 5_001; row++) { insert.run( - normalizePathForSql(path.join(root, "src", "a.ts")), + normalizePathForSql(path.join(root, "src", "a.ts"), root), `seed-${row}`, "current", 2, @@ -380,11 +380,13 @@ describe("disk cache uses sqlite backend", () => { const db = new DatabaseSync(duplicateCacheDbPath(root)); const row = db .prepare("SELECT version, payload FROM duplicate_unit_cache WHERE file = ?") - .get(normalizePathForSql(path.join(root, "src", "a.ts"))) as { version: number; payload: Uint8Array } | undefined; + .get(normalizePathForSql(path.join(root, "src", "a.ts"), root)) as + | { version: number; payload: Uint8Array } + | undefined; db.close(); expect(row).toBeDefined(); - expect(row!.version).toBe(3); + expect(row!.version).toBe(4); const decompressed = brotliDecompressSync(row!.payload).toString("utf8"); const units = JSON.parse(decompressed) as Array>; expect(units.length).toBeGreaterThan(0); @@ -400,7 +402,7 @@ describe("disk cache uses sqlite backend", () => { await findDuplicates(index, { minConfidence: "high", limit: 5 }); closeDuplicateUnitCacheDatabase(root); - const aFile = normalizePathForSql(path.join(root, "src", "a.ts")); + const aFile = normalizePathForSql(path.join(root, "src", "a.ts"), root); const staleDb = new DatabaseSync(duplicateCacheDbPath(root)); staleDb .prepare("UPDATE duplicate_unit_cache SET version = 2, payload = ? WHERE file = ?") @@ -416,6 +418,6 @@ describe("disk cache uses sqlite backend", () => { | { version: number } | undefined; after.close(); - expect(row?.version).toBe(3); + expect(row?.version).toBe(4); }); }); From 75fa0b2da32e2c50586f16fdb28f1e9a30831c6d Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:27:21 -0400 Subject: [PATCH 03/28] test(cache): cover resolver and bloom invalidation --- src/indexer/build-index.ts | 22 ++++++++++++++--- tests/agent-session.test.ts | 33 +++++++++++++++++++++++++ tests/cache-invalidation.test.ts | 12 ++++------ tests/node-modules-and-paths.test.ts | 36 +++++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index c5f77fab..cdef18f2 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -907,7 +907,12 @@ async function buildIndexFromFileListShared( manifestEntries: manifestEntriesForIndex, }); if (manifestEntries) { - await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries, projectRoot)); + await writeProjectIndexSnapshot( + projectRoot, + opts, + index, + projectSnapshotFilesSignature(manifestEntries, projectRoot), + ); } if (buildStartedAt !== undefined) { emitIndexLifecycleProgress(opts, "complete", "build", index.byFile.size, performance.now() - buildStartedAt); @@ -1326,6 +1331,10 @@ export async function buildProjectIndexIncremental( }; } const changedFiles = new Set(); + const forceNodeModuleReResolution = normalizeGraphOptions(opts?.graph).resolveNodeModules; + if (forceNodeModuleReResolution) { + for (const file of allFiles) changedFiles.add(file); + } const markAsChanged = (file: string): void => { if (allFiles.has(file)) changedFiles.add(file); }; @@ -1571,7 +1580,9 @@ export async function buildProjectIndexIncremental( } expandStarImports(modules, opts); const retainedTrackedEntries = Object.entries(trackedEntries).filter(([file]) => !deletedTrackedFiles.has(file)); - const cachedGraphEntries = new Map(retainedTrackedEntries); + const cachedGraphEntries = normalizeGraphOptions(opts?.graph).resolveNodeModules + ? new Map() + : new Map(retainedTrackedEntries); const manifestEntries = new Map(cachedGraphEntries); const baseGraph: Graph | undefined = cachedGraphEntries.size > 0 ? { nodes: new Set(), edges: [] } : undefined; @@ -1646,7 +1657,12 @@ export async function buildProjectIndexIncremental( manifestEntries: projectIndexManifestEntries(manifestEntries), buildReport: report, }); - await writeProjectIndexSnapshot(projectRoot, opts, index, projectSnapshotFilesSignature(manifestEntries, projectRoot)); + await writeProjectIndexSnapshot( + projectRoot, + opts, + index, + projectSnapshotFilesSignature(manifestEntries, projectRoot), + ); if (updateStartedAt !== undefined) { emitIndexLifecycleProgress(opts, "complete", "update", index.byFile.size, performance.now() - updateStartedAt); } else { diff --git a/tests/agent-session.test.ts b/tests/agent-session.test.ts index 3b45ecb6..2ac5a583 100644 --- a/tests/agent-session.test.ts +++ b/tests/agent-session.test.ts @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildProjectIndexIncremental, type BuildReport } from "../src/index.js"; import { AGENT_FRESHNESS_CHECK_INTERVAL_MS, createAgentSession, listAgentSessionFiles } from "../src/agent/session.js"; import * as symbolGraphBuild from "../src/graphs/symbol-graph-detailed.js"; import * as indexerBuild from "../src/indexer/build-index.js"; @@ -458,6 +459,38 @@ describe("agent session", () => { expect(refreshed.version).toBe(3); }); + it("invalidates module, project snapshot, and detailed sidecar on core epoch drift", async () => { + const root = await mkGitRepo(); + const initial = await createAgentSession({ root }).loadProject(); + const cacheDir = path.join(root, ".codegraph-cache", "index-v1"); + const manifestPath = path.join(cacheDir, "manifest.json"); + const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")) as { + buildOptions?: { coreAlgorithmEpoch?: number; implementationFingerprint?: string }; + }; + manifest.buildOptions = { + ...(manifest.buildOptions ?? {}), + coreAlgorithmEpoch: 0, + implementationFingerprint: "0".repeat(64), + }; + await fs.writeFile(manifestPath, JSON.stringify(manifest), "utf8"); + const snapshotPath = projectSnapshotPath(root); + const snapshot = JSON.parse(brotliDecompressSync(await fs.readFile(snapshotPath)).toString("utf8")) as { + implementationFingerprint?: string; + }; + snapshot.implementationFingerprint = "0".repeat(64); + await fs.writeFile(snapshotPath, brotliCompressSync(JSON.stringify(snapshot))); + const sidecarPath = detailedSymbolGraphSnapshotPath(root); + const sidecar = (await readDetailedSidecar(sidecarPath)) as MutableDetailedSymbolGraphSidecar; + sidecar.implementationFingerprint = "0".repeat(64); + await writeDetailedSidecar(sidecarPath, sidecar); + const report: BuildReport = { timings: {} }; + await buildProjectIndexIncremental(root, { cache: "disk", threads: 1, report }); + expect(report.files?.parsed ?? 0).toBeGreaterThan(0); + const symbolGraphSpy = vi.spyOn(symbolGraphBuild, "buildSymbolGraphDetailed"); + await createAgentSession({ root }).loadProject(); + expect(symbolGraphSpy).toHaveBeenCalledTimes(1); + }); + it("invalidates the detailed sidecar after a tracked edit", async () => { const root = await mkGitRepo(); const initial = await createAgentSession({ root }).loadProject(); diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index c975cee8..1c78b03c 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -1789,7 +1789,7 @@ describe("Cache invalidation and strict hashing", () => { expect(incremental.bloomFilters).toBeUndefined(); }); - it("falls back when project snapshot bloom filters are malformed", async () => { + it("rejects same-length invalid-base64 bloom payloads", async () => { const root = await mkTmpDir("dg-snapshot-bloom-malformed-"); const entryPath = path.join(root, "entry.ts"); await fsp.writeFile(entryPath, "export const guarded = 1;\n", "utf8"); @@ -1797,14 +1797,12 @@ describe("Cache invalidation and strict hashing", () => { await buildProjectIndex(root, { threads: 2, cache: "disk", useBloomFilters: true }); const snapshotPath = projectSnapshotPathFor(root); const snapshot = (await readProjectSnapshot(snapshotPath)) as { - bloomFilters?: Record; + bloomFilters?: Record; }; + const [key, original] = Object.entries(snapshot.bloomFilters ?? {})[0] ?? []; + if (!key || !original?.bitsBase64) throw new Error("missing persisted bloom filter"); snapshot.bloomFilters = { - [normalize(entryPath)]: { - size: 1_000, - hashCount: 3, - bitsBase64: "AAAA", - }, + [key]: { ...original, bitsBase64: `!${original.bitsBase64.slice(1)}` }, }; await writeProjectSnapshot(snapshotPath, snapshot); diff --git a/tests/node-modules-and-paths.test.ts b/tests/node-modules-and-paths.test.ts index 0626f7dc..b325cc39 100644 --- a/tests/node-modules-and-paths.test.ts +++ b/tests/node-modules-and-paths.test.ts @@ -1,8 +1,9 @@ import { describe, it, expect } from "vitest"; import path from "node:path"; import fsp from "node:fs/promises"; -import { collectGraph } from "../src/index.js"; +import { buildProjectIndex, buildProjectIndexIncremental, type BuildReport } from "../src/index.js"; import { resolveFromNodeModules } from "../src/util/resolution/node.js"; +import { collectGraph } from "../src/index.js"; import { mkTmpDir, normalizeTestPath } from "./helpers/filesystem.js"; describe("Node modules resolution (opt-in) and path normalization", () => { @@ -69,6 +70,39 @@ describe("Node modules resolution (opt-in) and path normalization", () => { ); }); + it("refreshes cached node-module edges when package targets change", async () => { + const root = await mkTmpDir("dg-nm-incremental-"); + const nm = path.join(root, "node_modules", "my-pkg"); + const main = path.join(root, "main.js"); + await fsp.mkdir(nm, { recursive: true }); + await fsp.writeFile(main, 'import "my-pkg";\n', "utf8"); + await fsp.writeFile(path.join(nm, "first.js"), "module.exports = 1;\n", "utf8"); + await fsp.writeFile(path.join(nm, "second.js"), "module.exports = 2;\n", "utf8"); + const packagePath = path.join(nm, "package.json"); + await fsp.writeFile(packagePath, JSON.stringify({ name: "my-pkg", main: "first.js" }), "utf8"); + + const first = await buildProjectIndex(root, { + cache: "disk", + graph: { resolveNodeModules: true }, + threads: 1, + }); + expect(first.graph.edges.some((edge) => edge.to.type === "file" && edge.to.path.endsWith("/first.js"))).toBe(true); + + await fsp.writeFile(packagePath, JSON.stringify({ name: "my-pkg", main: "second.js" }), "utf8"); + const report: BuildReport = { timings: {} }; + const second = await buildProjectIndexIncremental(root, { + cache: "disk", + graph: { resolveNodeModules: true }, + threads: 1, + report, + }); + expect(second.graph.edges.map((edge) => (edge.to.type === "file" ? edge.to.path : edge.to.name))).toEqual([ + expect.stringContaining("second.js"), + ]); + expect(second.graph.edges.some((edge) => edge.to.type === "file" && edge.to.path.endsWith("/first.js"))).toBe( + false, + ); + }); it("normalizes paths to forward slashes in nodes and edges", async () => { const root = await mkTmpDir("dg-paths-"); const a = path.join(root, "a.ts"); From 2fed86f7c313a7bbad0677c4595a5f7415766069 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 12:36:35 -0400 Subject: [PATCH 04/28] fix(cache): preserve duplicate consumer path contract --- src/duplicates/unitCache.ts | 31 +++++++++++++++++++------------ src/duplicates/units.ts | 4 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index b4d91736..7cc0c732 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -157,10 +157,14 @@ export function normalizedDuplicateUnitCacheNativeMode( return nativeMode; } -export function duplicateUnitCacheSignature(index: ProjectIndex, file: string): string | undefined { +export function duplicateUnitCacheSignature( + index: ProjectIndex, + file: string, + projectRoot?: string, +): string | undefined { + const root = projectRoot ?? index.projectRoot; const entry = - index.manifestEntries?.get(file) ?? - (index.projectRoot ? index.manifestEntries?.get(cacheRelativePath(index.projectRoot, file)) : undefined); + index.manifestEntries?.get(file) ?? (root ? index.manifestEntries?.get(cacheRelativePath(root, file)) : undefined); return entry?.gitSig ?? entry?.sig; } @@ -338,8 +342,9 @@ export function tryLoadDuplicateUnitsFromCache( index: ProjectIndex, file: string, variant: string, + projectRoot?: string, ): DuplicateInternalUnit[] | null { - const sig = duplicateUnitCacheSignature(index, file); + const sig = duplicateUnitCacheSignature(index, file, projectRoot); if (!sig) return null; const key = duplicateUnitCacheKey(file, variant); if (index.cacheMode === "memory") { @@ -349,15 +354,15 @@ export function tryLoadDuplicateUnitsFromCache( if (index.cacheMode !== "disk") return null; try { const entry = duplicateUnitDiskCache(index); - const root = index.projectRoot ?? ""; - const relativeFile = cacheRelativePath(root, file); + const root = projectRoot ?? index.projectRoot ?? ""; + const relativeFile = root ? cacheRelativePath(root, file) : file; const row = entry?.statements?.load.get(relativeFile, variant) as | { sig: string; version: number; payload: Uint8Array } | undefined; if (!row || row.sig !== sig || row.version !== DUPLICATE_UNIT_CACHE_VERSION) return null; const parsed = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as unknown; if (!Array.isArray(parsed)) return null; - return deserializeDuplicateUnits(transformDuplicateUnits(root, parsed, false)); + return deserializeDuplicateUnits(root ? transformDuplicateUnits(root, parsed, false) : parsed); } catch { return null; } @@ -369,8 +374,9 @@ export function writeDuplicateUnitsToCache( file: string, variant: string, units: DuplicateInternalUnit[], + projectRoot?: string, ): void { - const sig = duplicateUnitCacheSignature(index, file); + const sig = duplicateUnitCacheSignature(index, file, projectRoot); if (!sig) return; const key = duplicateUnitCacheKey(file, variant); if (index.cacheMode === "memory") { @@ -379,16 +385,17 @@ export function writeDuplicateUnitsToCache( } if (index.cacheMode === "disk") { try { + const root = projectRoot ?? index.projectRoot ?? ""; const entry = duplicateUnitDiskCache(index); - const root = index.projectRoot ?? ""; + const serialized = serializeDuplicateUnits(units); const payload = brotliCompressSync( - JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(units), true)), + JSON.stringify(root ? transformDuplicateUnits(root, serialized, true) : serialized), { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, }, ); entry?.statements?.write.run( - cacheRelativePath(root, file), + root ? cacheRelativePath(root, file) : file, variant, sig, DUPLICATE_UNIT_CACHE_VERSION, @@ -434,7 +441,7 @@ function transformDuplicateUnits( ): DuplicateSerializedUnit[] { return units.map((unit) => ({ ...unit, - file: transformDuplicatePath(root, unit.file, toRelative), + file: cacheRelativePath(root, unit.file), absoluteFile: transformDuplicatePath(root, unit.absoluteFile, toRelative), handle: transformDuplicatePath(root, unit.handle, toRelative), fileHandle: transformDuplicatePath(root, unit.fileHandle, toRelative), diff --git a/src/duplicates/units.ts b/src/duplicates/units.ts index 933b4bf7..8c10a8d6 100644 --- a/src/duplicates/units.ts +++ b/src/duplicates/units.ts @@ -508,7 +508,7 @@ export async function collectDuplicateUnits( const belowThresholdUnitsByFile = new Map(); for (const file of normalizedFiles) { - const cachedUnits = tryLoadDuplicateUnitsFromCache(index, file, variant); + const cachedUnits = tryLoadDuplicateUnitsFromCache(index, file, variant, options.projectRoot); const fileUnits = cachedUnits ?? (await buildDuplicateUnitsForFile( @@ -522,7 +522,7 @@ export async function collectDuplicateUnits( astContextCache, )); if (!cachedUnits) { - writeDuplicateUnitsToCache(index, file, variant, fileUnits); + writeDuplicateUnitsToCache(index, file, variant, fileUnits, options.projectRoot); } for (const unit of fileUnits) { if (!shouldKeepUnit(unit, options.includeSmall, options.minTokens)) { From e295e690569587271c9d54b59d1692caf70842e7 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 22:59:47 -0400 Subject: [PATCH 05/28] fix(cache): preserve portable path identities --- src/duplicates/unitCache.ts | 53 +++++++++++++++++--------- src/indexer/build-cache/manifest.ts | 35 +++++++++++++---- src/indexer/build-manifest.ts | 14 +++++-- src/indexer/types.ts | 2 +- tests/cache-invalidation.test.ts | 40 +++++++++++++++++-- tests/disk-cache-sqlite.test.ts | 59 +++++++++++++++++++++++++++++ 6 files changed, 171 insertions(+), 32 deletions(-) diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 7cc0c732..4783a441 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -424,14 +424,27 @@ export function serializeDuplicateUnits(units: DuplicateInternalUnit[]): Duplica signatures: [...unit.signatures], })); } -function transformDuplicatePath(root: string, value: string, toRelative: boolean): string { - const separator = value.indexOf("::"); - if (separator >= 0) { - const file = value.slice(0, separator); - const resolved = toRelative ? cacheRelativePath(root, file) : cacheAbsolutePath(root, file); - return `${resolved}${value.slice(separator)}`; +function transformDuplicateHandle(root: string, value: string): string { + const parts = value.split(":"); + let filePartIndex = -1; + if (parts[0] === "file" || parts[0] === "chunk") { + filePartIndex = 1; + } else if (parts[0] === "sql" || parts[0] === "symbol") { + filePartIndex = 2; } - return toRelative ? cacheRelativePath(root, value) : cacheAbsolutePath(root, value); + if (filePartIndex < 0 || parts.length <= filePartIndex) return value; + const encodedFile = parts[filePartIndex]; + if (encodedFile === undefined) return value; + try { + parts[filePartIndex] = encodeURIComponent(cacheRelativePath(root, decodeURIComponent(encodedFile))); + return parts.join(":"); + } catch { + return value; + } +} + +function duplicateUnitId(unit: DuplicateSerializedUnit, absoluteFile: string): string { + return `${normalizePath(absoluteFile)}:${unit.startLine}:${unit.endLine}:${unit.kind}:${unit.name ?? ""}`; } function transformDuplicateUnits( @@ -439,16 +452,22 @@ function transformDuplicateUnits( units: DuplicateSerializedUnit[], toRelative: boolean, ): DuplicateSerializedUnit[] { - return units.map((unit) => ({ - ...unit, - file: cacheRelativePath(root, unit.file), - absoluteFile: transformDuplicatePath(root, unit.absoluteFile, toRelative), - handle: transformDuplicatePath(root, unit.handle, toRelative), - fileHandle: transformDuplicatePath(root, unit.fileHandle, toRelative), - ...(unit.sqlHandle ? { sqlHandle: transformDuplicatePath(root, unit.sqlHandle, toRelative) } : {}), - chunkHandle: transformDuplicatePath(root, unit.chunkHandle, toRelative), - ...(unit.symbolHandle ? { symbolHandle: transformDuplicatePath(root, unit.symbolHandle, toRelative) } : {}), - })); + return units.map((unit) => { + const absoluteFile = toRelative + ? cacheRelativePath(root, unit.absoluteFile) + : cacheAbsolutePath(root, unit.absoluteFile); + return { + ...unit, + file: cacheRelativePath(root, unit.file), + absoluteFile, + id: duplicateUnitId(unit, absoluteFile), + handle: transformDuplicateHandle(root, unit.handle), + fileHandle: transformDuplicateHandle(root, unit.fileHandle), + ...(unit.sqlHandle ? { sqlHandle: transformDuplicateHandle(root, unit.sqlHandle) } : {}), + chunkHandle: transformDuplicateHandle(root, unit.chunkHandle), + ...(unit.symbolHandle ? { symbolHandle: transformDuplicateHandle(root, unit.symbolHandle) } : {}), + }; + }); } export function isDuplicateSerializedUnit(value: unknown): value is DuplicateSerializedUnit { diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 82530197..013e3bce 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -16,12 +16,7 @@ import { import { assertFilePathWithinRoot, fileIdentityKey, isFilePathWithinRoot } from "../../util/paths.js"; import { getGitBlobHashes } from "../../util/git.js"; import { stringifyUnknown } from "../../util/ast.js"; -import { - cacheAbsolutePath, - cacheRelativePath, - cacheRoot, - fileSignature, -} from "./module-cache.js"; +import { cacheAbsolutePath, cacheRelativePath, cacheRoot, fileSignature } from "./module-cache.js"; import type { BuildOptions } from "../types.js"; import type { ManifestBuildOptions } from "./options.js"; @@ -133,7 +128,9 @@ export function transformManifestEntries( edge.to.type === "file" ? { ...edge.to, - path: toRelative ? cacheRelativePath(projectRoot, edge.to.path) : cacheAbsolutePath(projectRoot, edge.to.path), + path: toRelative + ? cacheRelativePath(projectRoot, edge.to.path) + : cacheAbsolutePath(projectRoot, edge.to.path), } : edge.to, })), @@ -179,6 +176,24 @@ export function sanitizeManifestTransientFilesForRoot(projectRoot: string, files return [...sanitizedFiles]; } +function resolveManifestSymlinkDirectories( + projectRoot: string, + storedProjectRoot: string, + directories: unknown, +): string[] | undefined { + if (directories === undefined) return undefined; + if (!Array.isArray(directories)) return undefined; + const resolvedDirectories = new Set(); + for (const directory of directories) { + if (typeof directory !== "string") continue; + const storedDirectory = cacheAbsolutePath(storedProjectRoot, directory); + if (!isFilePathWithinRoot(storedProjectRoot, storedDirectory)) continue; + const relativeDirectory = cacheRelativePath(storedProjectRoot, storedDirectory); + resolvedDirectories.add(cacheAbsolutePath(projectRoot, relativeDirectory)); + } + return [...resolvedDirectories]; +} + export async function computeConfigHash(projectRoot: string, logLevel?: LogLevel): Promise { try { const configFiles = await fg([...DEFAULT_PROJECT_MANIFESTS, CODEGRAPH_CONFIG_FILE, "**/.gitignore"], { @@ -272,11 +287,17 @@ export async function loadManifest(projectRoot: string, opts?: BuildOptions): Pr const oldFiles = parsed.files ?? {}; const relativeFiles = parsed.version === 3 ? transformManifestEntries(parsed.projectRoot, oldFiles, true) : oldFiles; + const symlinkDirectories = resolveManifestSymlinkDirectories( + projectRoot, + parsed.projectRoot, + parsed.symlinkDirectories, + ); const migrated: IndexManifest = { ...parsed, version: MANIFEST_VERSION, files: transformManifestEntries(projectRoot, relativeFiles, false), transientFiles: sanitizeManifestTransientFilesForRoot(projectRoot, parsed.transientFiles), + ...(symlinkDirectories !== undefined ? { symlinkDirectories } : {}), }; return migrated; } catch { diff --git a/src/indexer/build-manifest.ts b/src/indexer/build-manifest.ts index 1a083228..13992028 100644 --- a/src/indexer/build-manifest.ts +++ b/src/indexer/build-manifest.ts @@ -11,7 +11,7 @@ import { type IndexManifest, type ManifestFileEntry, } from "./build-cache.js"; -import { pruneDiskModuleCache } from "./build-cache/module-cache.js"; +import { cacheRelativePath, pruneDiskModuleCache } from "./build-cache/module-cache.js"; import type { GraphCacheEntry, GraphBuildOptions } from "../graphs/types.js"; import type { BuildOptions, BuildReport, ManifestReport } from "./types.js"; @@ -51,8 +51,16 @@ export async function writeIndexManifestSnapshot(args: { graphOptions: args.graphOptions, buildOptions: summarizeBuildOptions(args.opts), files: transformManifestEntries(args.projectRoot, files, true), - transientFiles: (args.transientFiles ?? []).map((file) => path.relative(args.projectRoot, file).replace(/\\/g, "/")), - ...(args.symlinkDirectories !== undefined ? { symlinkDirectories: args.symlinkDirectories } : {}), + transientFiles: (args.transientFiles ?? []).map((file) => + path.relative(args.projectRoot, file).replace(/\\/g, "/"), + ), + ...(args.symlinkDirectories !== undefined + ? { + symlinkDirectories: args.symlinkDirectories.map((directory) => + cacheRelativePath(args.projectRoot, directory), + ), + } + : {}), }; const manifestWritten = await writeManifest(args.projectRoot, args.opts, manifestData); if (manifestWritten) pruneDiskModuleCache(args.projectRoot, Object.keys(files), args.opts); diff --git a/src/indexer/types.ts b/src/indexer/types.ts index 342da7d0..37ac0ed0 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -136,7 +136,7 @@ export type ProjectIndex = { */ export type LanguageExtensionMap = import("../languages.js").LanguageExtensionMap; -export type CacheLocation = "project" | "repo" | "user" | string; +export type CacheLocation = string; export type BuildOptions = { onProgress?: ((progress: ProgressUpdate) => void) | undefined; diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 1c78b03c..48da4891 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -2225,8 +2225,7 @@ describe("Cache invalidation and strict hashing", () => { await buildProjectIndex(root, { cache: "disk" }); const manifest = await readManifest(root); - expect(manifest.symlinkDirectories).toBeDefined(); - expect((manifest.symlinkDirectories ?? []).map(normalize)).toContain(normalize(linkedPackage)); + expect(manifest.symlinkDirectories).toContain("linked-core"); }); it("prunes stale symlink directory hints from the manifest after warm re-verification", async () => { @@ -2245,7 +2244,7 @@ describe("Cache invalidation and strict hashing", () => { await buildProjectIndex(root, { cache: "disk" }); const staleManifest = await readManifest(root); - expect((staleManifest.symlinkDirectories ?? []).map(normalize)).toContain(normalize(linkedPackage)); + expect(staleManifest.symlinkDirectories).toContain("linked-core"); await fsp.rm(linkedPackage, { recursive: true, force: true }); await buildProjectIndex(root, { cache: "disk" }); @@ -2281,7 +2280,7 @@ describe("Cache invalidation and strict hashing", () => { expect(rebuilt.byFile.has(fileIdentityKey(normalize(path.join(linkedPackage, "src", "index.ts"))))).toBe(true); const refreshedManifest = await readManifest(root); - expect((refreshedManifest.symlinkDirectories ?? []).map(normalize)).toContain(normalize(linkedPackage)); + expect(refreshedManifest.symlinkDirectories).toContain("linked-core"); }); it("persists an empty symlinkDirectories list for projects without symlinks", async () => { @@ -2326,6 +2325,39 @@ describe("Cache invalidation and strict hashing", () => { expect(report.files?.cached).toBeGreaterThan(0); }); + it("reuses symlink directory hints after moving a project tree", async () => { + const sourceRoot = await mkTmpDir("dg-cache-move-symlink-source-"); + const movedRoot = `${sourceRoot}-moved`; + const sourcePackage = path.join(sourceRoot, "packages", "core"); + const sourceLink = path.join(sourceRoot, "linked-core"); + await fsp.mkdir(sourcePackage, { recursive: true }); + await fsp.writeFile(path.join(sourcePackage, "entry.ts"), "export const entry = 1;\n", "utf8"); + + try { + await fsp.symlink(sourcePackage, sourceLink, "junction"); + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "EPERM") return; + throw error; + } + + await buildProjectIndex(sourceRoot, { cache: "disk", threads: 1 }); + const persisted = await readManifest(sourceRoot); + expect(persisted.symlinkDirectories).toEqual(["linked-core"]); + + await fsp.rename(sourceRoot, movedRoot); + const movedPackage = path.join(movedRoot, "packages", "core"); + const movedLink = path.join(movedRoot, "linked-core"); + await fsp.rm(movedLink, { recursive: true, force: true }); + await fsp.symlink(movedPackage, movedLink, "junction"); + + const report: BuildReport = { timings: {} }; + const moved = await buildProjectIndexIncremental(movedRoot, { cache: "disk", threads: 1, report }); + + expect(moved.byFile.has(fileIdentityKey(normalize(path.join(movedLink, "entry.ts"))))).toBe(true); + expect(report.cache?.misses ?? 0).toBe(0); + expect(report.files?.cached).toBeGreaterThan(0); + }); + it("namespaces cache roots under repository anchors and accepts a git file", async () => { const repoRoot = await mkTmpDir("dg-cache-anchor-repo-"); const projectRoot = path.join(repoRoot, "packages", "app"); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 5c3cbdec..f3f6eb8c 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -6,6 +6,8 @@ import { brotliDecompressSync } from "node:zlib"; import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/index.js"; import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; +import { tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "../src/duplicates/unitCache.js"; +import { buildInternalUnit, formatDuplicateSqlHandle } from "../src/duplicates/units.js"; import { SqliteDatabase } from "../src/sqlite-driver.js"; import { mkTmpDir } from "./helpers/filesystem.js"; @@ -81,6 +83,22 @@ export function normalizeInvoiceRows(rows: Array<{ amount: number; tax: number } await fsp.writeFile(path.join(root, "src", "b.ts"), duplicateSource, "utf8"); } +async function writeDuplicateSqlProject(root: string): Promise { + const duplicateSource = ` +CREATE TABLE invoice_entries ( + id INTEGER PRIMARY KEY, + customer_id INTEGER NOT NULL, + subtotal_cents INTEGER NOT NULL, + tax_cents INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +`; + await fsp.mkdir(path.join(root, "schema"), { recursive: true }); + await fsp.writeFile(path.join(root, "schema", "a.sql"), duplicateSource, "utf8"); + await fsp.writeFile(path.join(root, "schema", "b.sql"), duplicateSource, "utf8"); +} + describe("disk cache uses sqlite backend", () => { it("persists module cache in sqlite and reuses entries", async () => { const root = await mkTmpDir("dg-disk-cache-"); @@ -395,6 +413,47 @@ describe("disk cache uses sqlite backend", () => { expect(() => JSON.parse(Buffer.from(row!.payload).toString("utf8"))).toThrow(); }); + it("persists SQL duplicate identities relative to the project root", async () => { + const root = await mkTmpDir("dg-disk-cache-portable-sql-duplicates-"); + await writeDuplicateSqlProject(root); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const file = normalizePathForSql(path.join(root, "schema", "a.sql")); + const source = await fsp.readFile(file, "utf8"); + const unit = buildInternalUnit( + { + file: "schema/a.sql", + startLine: 1, + endLine: 8, + languageId: "sql", + kind: "symbol", + name: "invoice_entries", + }, + file, + source, + 3, + 2, + index.nativeMode, + { sqlHandle: formatDuplicateSqlHandle("schema/a.sql", "invoice_entries", 1) }, + ); + writeDuplicateUnitsToCache(index, file, "portable-sql", [unit], root); + + const db = new DatabaseSync(duplicateCacheDbPath(root)); + const row = db.prepare("SELECT payload FROM duplicate_unit_cache WHERE file = ?").get("schema/a.sql") as + | { payload: Uint8Array } + | undefined; + db.close(); + + expect(row).toBeDefined(); + const units = JSON.parse(brotliDecompressSync(row!.payload).toString("utf8")) as Array>; + expect(units[0]?.sqlHandle).toBe(formatDuplicateSqlHandle("schema/a.sql", "invoice_entries", 1)); + expect(JSON.stringify(units[0])).not.toContain(normalizePathForSql(root)); + + const loaded = tryLoadDuplicateUnitsFromCache(index, file, "portable-sql", root); + expect(loaded?.[0]?.absoluteFile).toBe(normalizePathForSql(file)); + expect(loaded?.[0]?.id?.startsWith(`${normalizePathForSql(file)}:`)).toBe(true); + expect(loaded?.[0]?.sqlHandle).toBe(formatDuplicateSqlHandle("schema/a.sql", "invoice_entries", 1)); + }); + it("ignores duplicate cache rows written by an older payload version", async () => { const root = await mkTmpDir("dg-disk-cache-stale-duplicates-"); await writeDuplicateProject(root); From 4c700afd5c643dbf009238883c38702a981e2494 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 01:17:46 -0400 Subject: [PATCH 06/28] fix(cache): confine persisted path rehydration --- src/duplicates/unitCache.ts | 4 +- src/indexer/build-cache/module-cache.ts | 10 +- src/indexer/build-cache/project-snapshot.ts | 11 +- tests/cache-path-confinement.test.ts | 173 ++++++++++++++++++++ tests/disk-cache-sqlite.test.ts | 68 +++++++- 5 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 tests/cache-path-confinement.test.ts diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 4783a441..c74b1f4f 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -427,9 +427,9 @@ export function serializeDuplicateUnits(units: DuplicateInternalUnit[]): Duplica function transformDuplicateHandle(root: string, value: string): string { const parts = value.split(":"); let filePartIndex = -1; - if (parts[0] === "file" || parts[0] === "chunk") { + if (parts[0] === "file" || parts[0] === "chunk" || parts[0] === "symbol") { filePartIndex = 1; - } else if (parts[0] === "sql" || parts[0] === "symbol") { + } else if (parts[0] === "sql") { filePartIndex = 2; } if (filePartIndex < 0 || parts.length <= filePartIndex) return value; diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 5f2a3778..fefe3e4b 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -17,7 +17,7 @@ import { type SqliteTableColumn, } from "../../util/sqliteSchema.js"; import type { BuildOptions, BuildReport, ModuleIndex } from "../types.js"; -import { fileIdentityKey, normalizePath } from "../../util/paths.js"; +import { assertFilePathWithinRoot, fileIdentityKey, normalizePath } from "../../util/paths.js"; import { lruMapGet, lruMapSet } from "../../util/lruMap.js"; import { initCacheReport } from "./reports.js"; import { getImplementationFingerprint } from "./options.js"; @@ -433,7 +433,10 @@ function isModuleIndex(value: unknown): value is ModuleIndex { function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelative: boolean): ModuleIndex { const copy = structuredClone(module); - const transform = (file: string): string => (toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file)); + const transform = (file: string): string => + toRelative + ? cacheRelativePath(projectRoot, file) + : assertFilePathWithinRoot(projectRoot, cacheAbsolutePath(projectRoot, file), "Persisted cache path"); copy.file = transform(copy.file); for (const local of copy.locals) local.file = transform(local.file); for (const entry of copy.exports) { @@ -478,8 +481,9 @@ export function tryLoadFromCache( if (row && row.sig === sig && row.version === PARSED_CACHE_VERSION) { const parsed: unknown = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")); if (isModuleIndex(parsed)) { + const rehydrated = transformModulePaths(projectRoot, parsed, false); if (cacheEnabled && cacheReport) cacheReport.hits += 1; - return transformModulePaths(projectRoot, parsed, false); + return rehydrated; } } } catch (error) { diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 41906a6a..30ab042c 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -9,7 +9,7 @@ import type { ProjectFileInfo } from "../../util/projectFiles.js"; import { BloomFilter, BloomFilterCache } from "../../util/bloomFilter.js"; import { summarizeAnalysis } from "../../analysisSummary.js"; import type { AnalysisSummary } from "../../analysisSummary.js"; -import { fileIdentityKey, isFilePathWithinRoot, normalizePath } from "../../util/paths.js"; +import { assertFilePathWithinRoot, fileIdentityKey, isFilePathWithinRoot, normalizePath } from "../../util/paths.js"; import { getNativeRuntimeFingerprint } from "../../native/treeSitterNative.js"; import { SymbolKind } from "../types.js"; import type { @@ -152,7 +152,7 @@ function transformPath(root: string, value: string, toRelative: boolean): string if (toRelative) { return path.isAbsolute(value) ? cacheRelativePath(root, value) : value; } - return cacheAbsolutePath(root, value); + return assertFilePathWithinRoot(root, cacheAbsolutePath(root, value), "Persisted cache path"); } function transformHandle(root: string, value: string, toRelative: boolean): string { @@ -1076,8 +1076,13 @@ function deserializeBloomFilterCache( ): BloomFilterCache { const cache = new BloomFilterCache(); for (const [file, filter] of Object.entries(serialized)) { - cache.set( + const absoluteFile = assertFilePathWithinRoot( + projectRoot, cacheAbsolutePath(projectRoot, file), + "Persisted cache path", + ); + cache.set( + absoluteFile, BloomFilter.fromBuffer(Buffer.from(filter.bitsBase64, "base64"), filter.size, filter.hashCount), ); } diff --git a/tests/cache-path-confinement.test.ts b/tests/cache-path-confinement.test.ts new file mode 100644 index 00000000..0ae05334 --- /dev/null +++ b/tests/cache-path-confinement.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { brotliCompressSync, brotliDecompressSync } from "node:zlib"; +import { DatabaseSync } from "node:sqlite"; + +import { buildProjectIndex } from "../src/index.js"; +import type { ModuleIndex } from "../src/indexer/types.js"; +import { + cacheDatabasePath, + cacheRelativePath, + clearMemoryCache, + closeDiskCacheDatabase, + tryLoadFromCache, + writeToCache, +} from "../src/indexer/build-cache/module-cache.js"; +import { tryLoadPersistedBloomFilters } from "../src/indexer/build-cache/project-snapshot.js"; +import { mkTmpDir } from "./helpers/filesystem.js"; + +function moduleFor(file: string): ModuleIndex { + return { + file, + exports: [], + imports: [], + locals: [ + { file, localName: "value", kind: 1, range: { start: { line: 1, column: 0 }, end: { line: 1, column: 1 } } }, + ], + }; +} + +function snapshotPathFor(root: string): string { + return path.join(root, ".codegraph-cache", "index-v1", "project-index-snapshot.json"); +} + +describe("persisted cache rehydration is confined to the project root", () => { + it("rejects module cache rows whose persisted relative file path escapes the project root", async () => { + const root = await mkTmpDir("dg-confine-module-"); + const file = path.join(root, "a.ts"); + const sig = "sig-traversal"; + + writeToCache(root, file, sig, moduleFor(file), { cache: "disk" }); + closeDiskCacheDatabase(root, { cache: "disk" }); + + const dbPath = cacheDatabasePath(root, { cache: "disk" }, "index-cache.sqlite"); + const db = new DatabaseSync(dbPath); + try { + const maliciousModule: ModuleIndex = { + file: "../outside.ts", + exports: [], + imports: [], + locals: [], + }; + const maliciousPayload = brotliCompressSync(Buffer.from(JSON.stringify(maliciousModule))); + db.prepare("UPDATE module_cache SET payload = ? WHERE file = ?").run( + maliciousPayload, + cacheRelativePath(root, file), + ); + } finally { + db.close(); + } + + // The confinement helper must reject the row before its path is ever rehydrated into a + // module, so this must read back as a clean cache miss rather than an escaped absolute path. + expect(tryLoadFromCache(root, file, sig, { cache: "disk" })).toBeNull(); + + closeDiskCacheDatabase(root, { cache: "disk" }); + clearMemoryCache(); + }); + + it("rejects module cache imports whose resolved relative path escapes the project root", async () => { + const root = await mkTmpDir("dg-confine-module-import-"); + const file = path.join(root, "a.ts"); + const sig = "sig-traversal-import"; + + writeToCache(root, file, sig, moduleFor(file), { cache: "disk" }); + closeDiskCacheDatabase(root, { cache: "disk" }); + + const dbPath = cacheDatabasePath(root, { cache: "disk" }, "index-cache.sqlite"); + const db = new DatabaseSync(dbPath); + try { + const maliciousModule: ModuleIndex = { + file: "a.ts", + exports: [], + locals: [], + imports: [ + { + kind: "default", + local: "outside", + from: "../outside.ts", + resolved: "../../outside.ts", + }, + ], + }; + const maliciousPayload = brotliCompressSync(Buffer.from(JSON.stringify(maliciousModule))); + db.prepare("UPDATE module_cache SET payload = ? WHERE file = ?").run( + maliciousPayload, + cacheRelativePath(root, file), + ); + } finally { + db.close(); + } + + expect(tryLoadFromCache(root, file, sig, { cache: "disk" })).toBeNull(); + + closeDiskCacheDatabase(root, { cache: "disk" }); + clearMemoryCache(); + }); + + it("rejects a persisted project snapshot whose projectFiles escape the project root", async () => { + const root = await mkTmpDir("dg-confine-snapshot-projectfiles-"); + await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + + const snapshotPath = snapshotPathFor(root); + const raw = await fsp.readFile(snapshotPath); + const payload = JSON.parse(brotliDecompressSync(raw).toString("utf8")) as { + projectFiles?: Array>; + }; + payload.projectFiles = [ + { path: "../../outside.ts", kind: "file", type: "node", role: "manifest", projectRoot: "." }, + ]; + await fsp.writeFile(snapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + + // A confined rehydration must discard the whole snapshot rather than silently accepting an + // out-of-root entry, so the rebuild must not surface the smuggled path anywhere in the index. + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const projectFilePaths = (index.projectFiles ?? []).map((entry) => entry.path); + expect(projectFilePaths.some((filePath) => filePath.includes("outside.ts"))).toBe(false); + expect([...index.byFile.keys()].some((file) => file.endsWith("a.ts"))).toBe(true); + }); + + it("rejects a persisted project snapshot whose graph paths escape the project root", async () => { + const root = await mkTmpDir("dg-confine-snapshot-graph-"); + await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + + const snapshotPath = snapshotPathFor(root); + const raw = await fsp.readFile(snapshotPath); + const payload = JSON.parse(brotliDecompressSync(raw).toString("utf8")) as { + graph?: { nodes?: string[] }; + }; + if (!payload.graph) throw new Error("Expected the persisted snapshot graph."); + payload.graph.nodes = ["../../outside.ts"]; + await fsp.writeFile(snapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + + // Snapshot graph paths are rehydrated before the warm index is returned, so an escaped + // node rejects the snapshot and forces a root-confined rebuild. + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + expect([...index.graph.nodes].some((file) => file.includes("outside.ts"))).toBe(false); + expect([...index.byFile.keys()].some((file) => file.endsWith("a.ts"))).toBe(true); + }); + + it("rejects persisted bloom filters keyed by a path that escapes the project root", async () => { + const root = await mkTmpDir("dg-confine-bloom-"); + await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1, useBloomFilters: true }); + + const snapshotPath = snapshotPathFor(root); + const raw = await fsp.readFile(snapshotPath); + const payload = JSON.parse(brotliDecompressSync(raw).toString("utf8")) as { + bloomFilters?: Record; + }; + const sampleFilter = Object.values(payload.bloomFilters ?? {})[0]; + expect(sampleFilter).toBeDefined(); + payload.bloomFilters = { "../../outside.ts": sampleFilter }; + await fsp.writeFile(snapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + + // The bloom-only fast path reads this section directly, bypassing the whole-snapshot + // transform, so it must apply the same confinement check on its own before reuse. + const bloomFilters = await tryLoadPersistedBloomFilters(root, { cache: "disk" }); + expect(bloomFilters).toBeNull(); + }); +}); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index f3f6eb8c..6371cc36 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -7,7 +7,7 @@ import { brotliDecompressSync } from "node:zlib"; import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/index.js"; import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; import { tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "../src/duplicates/unitCache.js"; -import { buildInternalUnit, formatDuplicateSqlHandle } from "../src/duplicates/units.js"; +import { buildInternalUnit, formatDuplicateSqlHandle, formatDuplicateSymbolHandle } from "../src/duplicates/units.js"; import { SqliteDatabase } from "../src/sqlite-driver.js"; import { mkTmpDir } from "./helpers/filesystem.js"; @@ -454,6 +454,72 @@ describe("disk cache uses sqlite backend", () => { expect(loaded?.[0]?.sqlHandle).toBe(formatDuplicateSqlHandle("schema/a.sql", "invoice_entries", 1)); }); + it("persists symbol duplicate identities relative to the project root, canonicalizing the file component distinct from SQL handles", async () => { + const root = await mkTmpDir("dg-disk-cache-portable-symbol-duplicates-"); + await writeDuplicateProject(root); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const file = normalizePathForSql(path.join(root, "src", "a.ts")); + const source = await fsp.readFile(file, "utf8"); + const namedHandle = formatDuplicateSymbolHandle(file, "normalizeInvoiceRows", 1, 0); + const persistedNamedHandle = formatDuplicateSymbolHandle("src/a.ts", "normalizeInvoiceRows", 1, 0); + // A symbol handle is `symbol::::` (file at index 1), while a SQL + // handle is `sql:::` (file at index 2). Reusing the SQL index for symbol + // handles would rewrite the encoded *name* as if it were the file. + const emptyNameHandle = formatDuplicateSymbolHandle(file, "", 1, 0); + const persistedEmptyNameHandle = formatDuplicateSymbolHandle("src/a.ts", "", 1, 0); + + const namedUnit = buildInternalUnit( + { + file: "src/a.ts", + startLine: 1, + endLine: 8, + languageId: "typescript", + kind: "symbol", + name: "normalizeInvoiceRows", + }, + file, + source, + 3, + 2, + index.nativeMode, + { symbolHandle: namedHandle }, + ); + const emptyNameUnit = buildInternalUnit( + { + file: "src/a.ts", + startLine: 1, + endLine: 8, + languageId: "typescript", + kind: "symbol", + name: "", + }, + file, + source, + 3, + 2, + index.nativeMode, + { symbolHandle: emptyNameHandle }, + ); + writeDuplicateUnitsToCache(index, file, "portable-symbol", [namedUnit, emptyNameUnit], root); + + const db = new DatabaseSync(duplicateCacheDbPath(root)); + const row = db.prepare("SELECT payload FROM duplicate_unit_cache WHERE file = ?").get("src/a.ts") as + | { payload: Uint8Array } + | undefined; + db.close(); + + expect(row).toBeDefined(); + const units = JSON.parse(brotliDecompressSync(row!.payload).toString("utf8")) as Array>; + expect(units[0]?.symbolHandle).toBe(persistedNamedHandle); + expect(units[1]?.symbolHandle).toBe(persistedEmptyNameHandle); + expect(JSON.stringify(units)).not.toContain(normalizePathForSql(root)); + + const loaded = tryLoadDuplicateUnitsFromCache(index, file, "portable-symbol", root); + expect(loaded?.[0]?.absoluteFile).toBe(normalizePathForSql(file)); + expect(loaded?.[0]?.symbolHandle).toBe(persistedNamedHandle); + expect(loaded?.[1]?.symbolHandle).toBe(persistedEmptyNameHandle); + }); + it("ignores duplicate cache rows written by an older payload version", async () => { const root = await mkTmpDir("dg-disk-cache-stale-duplicates-"); await writeDuplicateProject(root); From 30d1090839a7fafe5aa9884542af3239c5567d63 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 01:35:14 -0400 Subject: [PATCH 07/28] fix: confine persisted cache paths --- src/duplicates/unitCache.ts | 4 +- src/indexer/build-cache/manifest.ts | 14 ++++++- tests/cache-invalidation.test.ts | 57 ++++++++++++++++++++++++++++- tests/disk-cache-sqlite.test.ts | 44 +++++++++++++++++++++- 4 files changed, 113 insertions(+), 6 deletions(-) diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index c74b1f4f..261e0f30 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -27,7 +27,7 @@ import type { DuplicateUnitDiskStatements, } from "./types.js"; import { lruMapGet } from "../util/lruMap.js"; -import { fileIdentityKey, normalizePath } from "../util/paths.js"; +import { assertFilePathWithinRoot, fileIdentityKey, normalizePath } from "../util/paths.js"; import { cacheAbsolutePath, cacheRelativePath } from "../indexer/build-cache/module-cache.js"; // v4: project-relative file fields and handles. @@ -455,7 +455,7 @@ function transformDuplicateUnits( return units.map((unit) => { const absoluteFile = toRelative ? cacheRelativePath(root, unit.absoluteFile) - : cacheAbsolutePath(root, unit.absoluteFile); + : assertFilePathWithinRoot(root, cacheAbsolutePath(root, unit.absoluteFile), "Persisted duplicate unit path"); return { ...unit, file: cacheRelativePath(root, unit.file), diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 013e3bce..2f74a670 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -123,14 +123,24 @@ export function transformManifestEntries( ...entry, edges: entry.edges.map((edge) => ({ ...edge, - from: toRelative ? cacheRelativePath(projectRoot, edge.from) : cacheAbsolutePath(projectRoot, edge.from), + from: toRelative + ? cacheRelativePath(projectRoot, edge.from) + : assertFilePathWithinRoot( + projectRoot, + cacheAbsolutePath(projectRoot, edge.from), + "Persisted manifest edge source", + ), to: edge.to.type === "file" ? { ...edge.to, path: toRelative ? cacheRelativePath(projectRoot, edge.to.path) - : cacheAbsolutePath(projectRoot, edge.to.path), + : assertFilePathWithinRoot( + projectRoot, + cacheAbsolutePath(projectRoot, edge.to.path), + "Persisted manifest edge target", + ), } : edge.to, })), diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 48da4891..f0ac8603 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -5,7 +5,13 @@ import fsp from "node:fs/promises"; import fs from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from "node:zlib"; -import { buildProjectIndex, buildProjectIndexIncremental, resolveExport, type BuildReport } from "../src/index.js"; +import { + buildProjectIndex, + buildProjectIndexFromFiles, + buildProjectIndexIncremental, + resolveExport, + type BuildReport, +} from "../src/index.js"; import type { ModuleIndex, ProjectIndex } from "../src/indexer/types.js"; import * as indexer from "../src/indexer.js"; import * as buildCache from "../src/indexer/build-cache.js"; @@ -1119,6 +1125,55 @@ describe("Cache invalidation and strict hashing", () => { expect(report.timings?.totalMs).toEqual(expect.any(Number)); }); + it("rejects manifest edges outside the project before probing or reusing them", async () => { + const root = await mkTmpDir("dg-manifest-edge-confinement-"); + const sourcePath = path.join(root, "source.ts"); + const dependencyPath = path.join(root, "dependency.ts"); + const outsideRoot = await mkTmpDir("dg-manifest-edge-outside-"); + const outsideSource = normalize(path.join(outsideRoot, "source.ts")); + const outsideDependency = normalize(path.join(outsideRoot, "dependency.ts")); + await fsp.writeFile(sourcePath, "import { value } from './dependency';\nexport { value };\n", "utf8"); + await fsp.writeFile(dependencyPath, "export const value = 1;\n", "utf8"); + await fsp.writeFile(outsideSource, "export const outsideSource = true;\n", "utf8"); + await fsp.writeFile(outsideDependency, "export const outsideDependency = true;\n", "utf8"); + + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const manifestPath = manifestPathFor(root); + const manifest = JSON.parse(await fsp.readFile(manifestPath, "utf8")) as IndexManifest; + const sourceEntry = manifest.files["source.ts"]; + if (!sourceEntry?.edges.length) throw new Error("expected a persisted source edge"); + sourceEntry.edges[0] = { + ...sourceEntry.edges[0], + from: normalize(path.relative(root, outsideSource)), + to: { type: "file", path: normalize(path.relative(root, outsideDependency)) }, + }; + await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8"); + + const existsSpy = vi.spyOn(incrementalPlan, "pathExists"); + const accessSpy = vi.spyOn(fsp, "access"); + try { + const rebuilt = await buildProjectIndexFromFiles(root, [sourcePath, dependencyPath], { + cache: "disk", + threads: 1, + }); + + expect(existsSpy).not.toHaveBeenCalledWith(outsideDependency); + expect(accessSpy).not.toHaveBeenCalledWith(outsideDependency); + expect(rebuilt.graph.edges).not.toContainEqual( + expect.objectContaining({ from: outsideSource, to: { type: "file", path: outsideDependency } }), + ); + expect(rebuilt.graph.edges).toContainEqual( + expect.objectContaining({ + from: normalize(sourcePath), + to: { type: "file", path: normalize(dependencyPath) }, + }), + ); + } finally { + existsSpy.mockRestore(); + accessSpy.mockRestore(); + } + }); + it("loads unchanged incremental indexes from a project snapshot", async () => { const root = await mkTmpDir("dg-incremental-project-snapshot-"); const filePath = path.join(root, "foo.ts"); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 6371cc36..bbe02381 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi } from "vitest"; import path from "node:path"; import fsp from "node:fs/promises"; import { DatabaseSync } from "node:sqlite"; -import { brotliDecompressSync } from "node:zlib"; +import { brotliCompressSync, brotliDecompressSync } from "node:zlib"; import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/index.js"; import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; @@ -520,6 +520,48 @@ describe("disk cache uses sqlite backend", () => { expect(loaded?.[1]?.symbolHandle).toBe(persistedEmptyNameHandle); }); + it("rejects duplicate units whose persisted absolute file escapes the project", async () => { + const root = await mkTmpDir("dg-disk-cache-duplicate-unit-confinement-"); + await writeDuplicateProject(root); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const file = normalizePathForSql(path.join(root, "src", "a.ts")); + const source = await fsp.readFile(file, "utf8"); + const unit = buildInternalUnit( + { + file: "src/a.ts", + startLine: 1, + endLine: 8, + languageId: "typescript", + kind: "symbol", + name: "normalizeInvoiceRows", + }, + file, + source, + 3, + 2, + index.nativeMode, + ); + writeDuplicateUnitsToCache(index, file, "confinement", [unit], root); + closeDuplicateUnitCacheDatabase(root); + + const db = new DatabaseSync(duplicateCacheDbPath(root)); + const row = db + .prepare("SELECT payload FROM duplicate_unit_cache WHERE file = ? AND variant = ?") + .get("src/a.ts", "confinement") as { payload: Uint8Array } | undefined; + if (!row) throw new Error("expected duplicate cache row"); + const units = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as Array>; + units[0] = { ...units[0], absoluteFile: "../outside.ts" }; + db.prepare("UPDATE duplicate_unit_cache SET payload = ? WHERE file = ? AND variant = ?").run( + brotliCompressSync(JSON.stringify(units)), + "src/a.ts", + "confinement", + ); + db.close(); + + const reopenedIndex = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + expect(tryLoadDuplicateUnitsFromCache(reopenedIndex, file, "confinement", root)).toBeNull(); + }); + it("ignores duplicate cache rows written by an older payload version", async () => { const root = await mkTmpDir("dg-disk-cache-stale-duplicates-"); await writeDuplicateProject(root); From 3e907f48567826bad9fb695c6d978043256c082c Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 01:55:38 -0400 Subject: [PATCH 08/28] fix cache path portability validation --- src/cli/options.ts | 2 + src/duplicates/unitCache.ts | 44 ++++++++++++ src/indexer/build-cache/manifest.ts | 4 +- src/indexer/build-cache/module-cache.ts | 14 ++-- src/indexer/build-cache/project-snapshot.ts | 15 ++-- tests/cache-invalidation.test.ts | 20 +++++- tests/cache-path-confinement.test.ts | 79 +++++++++++++++++++++ tests/cli-options-validation.test.ts | 9 +++ tests/disk-cache-sqlite.test.ts | 63 ++++++++++++++++ 9 files changed, 235 insertions(+), 15 deletions(-) diff --git a/src/cli/options.ts b/src/cli/options.ts index 36934293..cd2be979 100644 --- a/src/cli/options.ts +++ b/src/cli/options.ts @@ -15,6 +15,8 @@ const CLI_VALUE_OPTIONS = new Set([ "--native", "--cache", "--cache-dir", + "--changed-since", + "--git-base", "--git-head", "--symbols-detailed-scope", "--symbols-detailed-max-edges", diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 261e0f30..40f1e435 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -362,6 +362,7 @@ export function tryLoadDuplicateUnitsFromCache( if (!row || row.sig !== sig || row.version !== DUPLICATE_UNIT_CACHE_VERSION) return null; const parsed = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as unknown; if (!Array.isArray(parsed)) return null; + if (root && !validatePersistedDuplicateUnits(root, parsed)) return null; return deserializeDuplicateUnits(root ? transformDuplicateUnits(root, parsed, false) : parsed); } catch { return null; @@ -443,6 +444,49 @@ function transformDuplicateHandle(root: string, value: string): string { } } +function duplicateHandleFilePartIndex(value: string): number | null { + const parts = value.split(":"); + if (parts[0] === "file" && parts.length === 2) return 1; + if (parts[0] === "chunk" && parts.length === 3) return 1; + if (parts[0] === "symbol" && parts.length === 5) return 1; + if (parts[0] === "sql" && parts.length === 4) return 2; + return null; +} + +function hasPersistedDuplicateHandlePathWithinRoot(root: string, value: string): boolean { + const filePartIndex = duplicateHandleFilePartIndex(value); + const handlePrefix = value.split(":")[0]; + if (filePartIndex === null) { + return handlePrefix !== "file" && handlePrefix !== "chunk" && handlePrefix !== "symbol" && handlePrefix !== "sql"; + } + const encodedFile = value.split(":")[filePartIndex]; + if (!encodedFile) return false; + try { + assertFilePathWithinRoot( + root, + cacheAbsolutePath(root, decodeURIComponent(encodedFile)), + "Persisted duplicate handle path", + ); + return true; + } catch { + return false; + } +} + +function validatePersistedDuplicateUnits(root: string, value: unknown[]): value is DuplicateSerializedUnit[] { + return value.every((unit) => { + if (!isDuplicateSerializedUnit(unit)) return false; + try { + assertFilePathWithinRoot(root, cacheAbsolutePath(root, unit.file), "Persisted duplicate unit file"); + assertFilePathWithinRoot(root, cacheAbsolutePath(root, unit.absoluteFile), "Persisted duplicate unit path"); + } catch { + return false; + } + const handles = [unit.handle, unit.fileHandle, unit.chunkHandle, unit.symbolHandle, unit.sqlHandle]; + return handles.every((handle) => handle === undefined || hasPersistedDuplicateHandlePathWithinRoot(root, handle)); + }); +} + function duplicateUnitId(unit: DuplicateSerializedUnit, absoluteFile: string): string { return `${normalizePath(absoluteFile)}:${unit.startLine}:${unit.endLine}:${unit.kind}:${unit.name ?? ""}`; } diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 2f74a670..a188d5e7 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -118,7 +118,9 @@ export function transformManifestEntries( ): Record { const transformed: Record = {}; for (const [file, entry] of Object.entries(files)) { - const key = toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file); + const key = toRelative + ? cacheRelativePath(projectRoot, file) + : assertFilePathWithinRoot(projectRoot, cacheAbsolutePath(projectRoot, file), "Persisted manifest file key"); transformed[key] = { ...entry, edges: entry.edges.map((edge) => ({ diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index fefe3e4b..24caa82a 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -107,7 +107,9 @@ function isForbiddenAnchor(candidate: string): boolean { const resolved = path.resolve(candidate); const home = path.resolve(os.homedir()); const parsed = path.parse(resolved); - return fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root); + return ( + fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root) + ); } function findRepositoryAnchor(projectRoot: string): CacheAnchorResolution { @@ -168,9 +170,7 @@ export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { if (path.basename(configured) === namespace) return configured; return path.join(configured, namespace); } - const base = opts?.cacheLocation === "user" - ? resolveCodegraphUserCacheRoot() - : path.join(anchor, ".codegraph-cache"); + const base = opts?.cacheLocation === "user" ? resolveCodegraphUserCacheRoot() : path.join(anchor, ".codegraph-cache"); const candidate = path.join(path.resolve(base), "index-v1", namespace); if (!sameRoot && !opts?.cacheLocation) { const legacy = path.join(root, ".codegraph-cache", "index-v1"); @@ -440,7 +440,11 @@ function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelati copy.file = transform(copy.file); for (const local of copy.locals) local.file = transform(local.file); for (const entry of copy.exports) { - if (entry.type === "local") entry.target.file = transform(entry.target.file); + if (entry.type === "local") { + entry.target.file = transform(entry.target.file); + } else { + entry.fromModule = transform(entry.fromModule); + } } for (const binding of copy.imports) { if (typeof binding.resolved === "string") binding.resolved = transform(binding.resolved); diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 30ab042c..942a9012 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -35,7 +35,7 @@ import { cacheAbsolutePath, cacheRelativePath, cacheRoot } from "./module-cache. import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); -const PROJECT_SNAPSHOT_VERSION = 5; +const PROJECT_SNAPSHOT_VERSION = 6; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; @@ -168,7 +168,11 @@ function transformModule(root: string, module: ModuleIndex, toRelative: boolean) copy.file = file(copy.file); for (const local of copy.locals) local.file = file(local.file); for (const entry of copy.exports) { - if (entry.type === "local") entry.target.file = file(entry.target.file); + if (entry.type === "local") { + entry.target.file = file(entry.target.file); + } else { + entry.fromModule = file(entry.fromModule); + } } for (const binding of copy.imports) { if (typeof binding.resolved === "string") binding.resolved = file(binding.resolved); @@ -208,7 +212,7 @@ function transformSnapshotPaths( function migrateProjectSnapshotPayload(value: unknown, currentRoot: string): unknown { if (!value || typeof value !== "object" || Array.isArray(value)) return value; const payload = value as Partial; - if (payload.version !== 4 || typeof payload.projectRoot !== "string") return value; + if ((payload.version !== 4 && payload.version !== 5) || typeof payload.projectRoot !== "string") return value; const relative = transformSnapshotPaths(value as ProjectIndexSnapshotPayload, payload.projectRoot, true); const migrated = transformSnapshotPaths(relative, currentRoot, false); migrated.version = PROJECT_SNAPSHOT_VERSION; @@ -1117,10 +1121,7 @@ function isSerializedBloomFilter(value: unknown): value is SerializedBloomFilter } catch { return false; } - return ( - decoded.length === expectedBytes && - decoded.toString("base64") === filter.bitsBase64 - ); + return decoded.length === expectedBytes && decoded.toString("base64") === filter.bitsBase64; } function isModuleIndex(value: unknown): value is ModuleIndex { diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index f0ac8603..97de54e9 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -1911,7 +1911,7 @@ describe("Cache invalidation and strict hashing", () => { nativeRuntimeFingerprint?: string; implementationFingerprint?: string; }; - expect(rewrittenSnapshot.version).toBe(5); + expect(rewrittenSnapshot.version).toBe(6); expect(initial.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.bloomFilters?.get(normalize(entryPath))?.mightContain("versioned")).toBe(true); @@ -2369,13 +2369,29 @@ describe("Cache invalidation and strict hashing", () => { it("reuses relative caches after moving a project tree", async () => { const sourceRoot = await mkTmpDir("dg-cache-move-source-"); const movedRoot = `${sourceRoot}-moved`; - await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export const entry = 1;\n", "utf8"); + await fsp.writeFile(path.join(sourceRoot, "dependency.ts"), "export const dependency = 1;\n", "utf8"); + await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export { dependency } from './dependency';\n", "utf8"); await buildProjectIndex(sourceRoot, { cache: "disk", threads: 1 }); + const snapshot = (await readProjectSnapshot(projectSnapshotPathFor(sourceRoot))) as { + version?: number; + modules?: Array<{ file?: string; exports?: Array<{ type?: string; fromModule?: string }> }>; + }; + const entryModule = snapshot.modules?.find((module) => module.file === "entry.ts"); + const reexport = entryModule?.exports?.find((entry) => entry.type === "reexport"); + if (!reexport) throw new Error("expected persisted reexport"); + snapshot.version = 5; + reexport.fromModule = normalize(path.join(sourceRoot, "dependency.ts")); + await writeProjectSnapshot(projectSnapshotPathFor(sourceRoot), snapshot); await fsp.rename(sourceRoot, movedRoot); const report: BuildReport = { timings: {} }; const moved = await buildProjectIndexIncremental(movedRoot, { cache: "disk", threads: 1, report }); expect(moved.byFile.has(fileIdentityKey(normalize(path.join(movedRoot, "entry.ts"))))).toBe(true); + const resolved = resolveExport(moved, normalize(path.join(movedRoot, "entry.ts")), "dependency"); + expect(resolved?.kind).toBe("resolved"); + if (resolved?.kind === "resolved") { + expect(resolved.def.file).toBe(normalize(path.join(movedRoot, "dependency.ts"))); + } expect(report.cache?.misses ?? 0).toBe(0); expect(report.files?.cached).toBeGreaterThan(0); }); diff --git a/tests/cache-path-confinement.test.ts b/tests/cache-path-confinement.test.ts index 0ae05334..d46e0ced 100644 --- a/tests/cache-path-confinement.test.ts +++ b/tests/cache-path-confinement.test.ts @@ -14,6 +14,7 @@ import { tryLoadFromCache, writeToCache, } from "../src/indexer/build-cache/module-cache.js"; +import { loadManifest } from "../src/indexer/build-cache/manifest.js"; import { tryLoadPersistedBloomFilters } from "../src/indexer/build-cache/project-snapshot.js"; import { mkTmpDir } from "./helpers/filesystem.js"; @@ -106,6 +107,45 @@ describe("persisted cache rehydration is confined to the project root", () => { clearMemoryCache(); }); + it("rejects module cache reexports whose persisted target path escapes the project root", async () => { + const root = await mkTmpDir("dg-confine-module-reexport-"); + const file = path.join(root, "barrel.ts"); + const sig = "sig-traversal-reexport"; + + writeToCache(root, file, sig, moduleFor(file), { cache: "disk" }); + closeDiskCacheDatabase(root, { cache: "disk" }); + + const dbPath = cacheDatabasePath(root, { cache: "disk" }, "index-cache.sqlite"); + const db = new DatabaseSync(dbPath); + try { + const maliciousModule: ModuleIndex = { + file: "barrel.ts", + exports: [ + { + type: "reexport", + exportedAs: "outside", + fromModule: "../../outside.ts", + sourceSpecifier: "./outside", + }, + ], + imports: [], + locals: [], + }; + const maliciousPayload = brotliCompressSync(Buffer.from(JSON.stringify(maliciousModule))); + db.prepare("UPDATE module_cache SET payload = ? WHERE file = ?").run( + maliciousPayload, + cacheRelativePath(root, file), + ); + } finally { + db.close(); + } + + expect(tryLoadFromCache(root, file, sig, { cache: "disk" })).toBeNull(); + + closeDiskCacheDatabase(root, { cache: "disk" }); + clearMemoryCache(); + }); + it("rejects a persisted project snapshot whose projectFiles escape the project root", async () => { const root = await mkTmpDir("dg-confine-snapshot-projectfiles-"); await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); @@ -150,6 +190,45 @@ describe("persisted cache rehydration is confined to the project root", () => { expect([...index.byFile.keys()].some((file) => file.endsWith("a.ts"))).toBe(true); }); + it("rejects a persisted project snapshot whose reexport target escapes the project root", async () => { + const root = await mkTmpDir("dg-confine-snapshot-reexport-"); + await fsp.writeFile(path.join(root, "dependency.ts"), "export const dependency = 1;\n", "utf8"); + await fsp.writeFile(path.join(root, "barrel.ts"), "export { dependency } from './dependency';\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + + const snapshotPath = snapshotPathFor(root); + const raw = await fsp.readFile(snapshotPath); + const payload = JSON.parse(brotliDecompressSync(raw).toString("utf8")) as { + modules?: Array<{ file?: string; exports?: Array> }>; + }; + const barrel = payload.modules?.find((module) => module.file === "barrel.ts"); + const reexport = barrel?.exports?.find((entry) => entry.type === "reexport"); + if (!reexport) throw new Error("Expected the persisted barrel reexport."); + reexport.fromModule = "../../outside.ts"; + await fsp.writeFile(snapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const barrelModule = [...index.byFile.values()].find((module) => module.file.endsWith("barrel.ts")); + const rebuiltReexport = barrelModule?.exports.find((entry) => entry.type === "reexport"); + expect(rebuiltReexport?.type).toBe("reexport"); + if (rebuiltReexport?.type === "reexport") { + expect(rebuiltReexport.fromModule).not.toContain("outside.ts"); + } + }); + + it("rejects manifest file keys that escape the project root before cache probes", async () => { + const root = await mkTmpDir("dg-confine-manifest-key-"); + await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + + const manifestPath = path.join(root, ".codegraph-cache", "index-v1", "manifest.json"); + const manifest = JSON.parse(await fsp.readFile(manifestPath, "utf8")) as { files?: Record }; + manifest.files = { "../../outside.ts": { sig: "tampered", edges: [] } }; + await fsp.writeFile(manifestPath, JSON.stringify(manifest), "utf8"); + + expect(await loadManifest(root, { cache: "disk" })).toBeNull(); + }); + it("rejects persisted bloom filters keyed by a path that escapes the project root", async () => { const root = await mkTmpDir("dg-confine-bloom-"); await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); diff --git a/tests/cli-options-validation.test.ts b/tests/cli-options-validation.test.ts index 3fe823e4..33c11ce1 100644 --- a/tests/cli-options-validation.test.ts +++ b/tests/cli-options-validation.test.ts @@ -38,6 +38,15 @@ describe("CLI enum option parsers", () => { }); describe("parseCliArgs value-option guard", () => { + it("parses --changed-since and --git-base as valued Git range options", () => { + const review = parseCliArgs("review", ["--changed-since", "HEAD~1"]); + const graphDelta = parseCliArgs("graph-delta", ["--git-base", "HEAD~1", "--git-head", "HEAD"]); + + expect(review.options.get("--changed-since")).toEqual(["HEAD~1"]); + expect(graphDelta.options.get("--git-base")).toEqual(["HEAD~1"]); + expect(graphDelta.options.get("--git-head")).toEqual(["HEAD"]); + }); + it("does not consume a following flag as a value", () => { expect(() => parseCliArgs("graph", ["--threads", "--json"])).toThrow(/Missing value for --threads option/); }); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index bbe02381..0f5cbc21 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -562,6 +562,69 @@ describe("disk cache uses sqlite backend", () => { expect(tryLoadDuplicateUnitsFromCache(reopenedIndex, file, "confinement", root)).toBeNull(); }); + it("rejects duplicate units with malformed or out-of-root encoded handle paths", async () => { + const root = await mkTmpDir("dg-disk-cache-duplicate-handle-confinement-"); + await writeDuplicateProject(root); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const file = normalizePathForSql(path.join(root, "src", "a.ts")); + const source = await fsp.readFile(file, "utf8"); + const unit = buildInternalUnit( + { + file: "src/a.ts", + startLine: 1, + endLine: 8, + languageId: "typescript", + kind: "symbol", + name: "normalizeInvoiceRows", + }, + file, + source, + 3, + 2, + index.nativeMode, + { + sqlHandle: formatDuplicateSqlHandle("src/a.ts", "normalizeInvoiceRows", 1), + symbolHandle: formatDuplicateSymbolHandle("src/a.ts", "normalizeInvoiceRows", 1, 0), + }, + ); + writeDuplicateUnitsToCache(index, file, "handle-confinement", [unit], root); + closeDuplicateUnitCacheDatabase(root); + + const fieldValues: Array<[string, string]> = [ + ["file", "../outside.ts"], + ["handle", "sql:normalizeInvoiceRows:..%2Foutside.ts:1"], + ["fileHandle", "file:%E0%A4%A"], + ["chunkHandle", "chunk:..%2Foutside.ts:1"], + ["symbolHandle", "symbol:..%2Foutside.ts:normalizeInvoiceRows:1:0"], + ["sqlHandle", "sql:normalizeInvoiceRows:..%2Foutside.ts:1"], + ]; + const reopenedIndex = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + + for (const [field, value] of fieldValues) { + const db = new DatabaseSync(duplicateCacheDbPath(root)); + const row = db + .prepare("SELECT payload FROM duplicate_unit_cache WHERE file = ? AND variant = ?") + .get("src/a.ts", "handle-confinement") as { payload: Uint8Array } | undefined; + if (!row) throw new Error("expected duplicate cache row"); + const units = JSON.parse(brotliDecompressSync(row.payload).toString("utf8")) as Array>; + units[0] = { ...units[0], [field]: value }; + db.prepare("UPDATE duplicate_unit_cache SET payload = ? WHERE file = ? AND variant = ?").run( + brotliCompressSync(JSON.stringify(units)), + "src/a.ts", + "handle-confinement", + ); + db.close(); + + expect(tryLoadDuplicateUnitsFromCache(reopenedIndex, file, "handle-confinement", root)).toBeNull(); + + const reset = new DatabaseSync(duplicateCacheDbPath(root)); + reset + .prepare("UPDATE duplicate_unit_cache SET payload = ? WHERE file = ? AND variant = ?") + .run(row.payload, "src/a.ts", "handle-confinement"); + reset.close(); + } + }); + it("ignores duplicate cache rows written by an older payload version", async () => { const root = await mkTmpDir("dg-disk-cache-stale-duplicates-"); await writeDuplicateProject(root); From eeb0c5e9b60aa9aa6ee077c7737f8062778c96e8 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 02:13:53 -0400 Subject: [PATCH 09/28] fix cache external reexports --- src/indexer/build-cache/module-cache.ts | 34 +++++++++-- src/indexer/build-cache/project-snapshot.ts | 6 +- tests/cache-invalidation.test.ts | 2 +- tests/cache-path-confinement.test.ts | 62 ++++++++++++++++++++- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 24caa82a..047bb468 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -16,14 +16,14 @@ import { sqliteTableColumns, type SqliteTableColumn, } from "../../util/sqliteSchema.js"; -import type { BuildOptions, BuildReport, ModuleIndex } from "../types.js"; -import { assertFilePathWithinRoot, fileIdentityKey, normalizePath } from "../../util/paths.js"; +import type { BuildOptions, BuildReport, ExportEntry, ModuleIndex } from "../types.js"; +import { assertFilePathWithinRoot, fileIdentityKey, isFilePathWithinRoot, normalizePath } from "../../util/paths.js"; import { lruMapGet, lruMapSet } from "../../util/lruMap.js"; import { initCacheReport } from "./reports.js"; import { getImplementationFingerprint } from "./options.js"; -// v4: relative file keys and explicit cache-anchor policy. -const PARSED_CACHE_VERSION = 4; +// v5: external reexports preserve their unresolved module specifier. +const PARSED_CACHE_VERSION = 5; const MODULE_CACHE_SCHEMA_VERSION = 2; const MODULE_CACHE_TABLE = "module_cache"; const MODULE_CACHE_SCHEMA_VERSION_KEY = "module_cache.schema_version"; @@ -431,6 +431,30 @@ function isModuleIndex(value: unknown): value is ModuleIndex { ); } +export function transformPersistedExportFromModule( + projectRoot: string, + entry: Exclude, + toRelative: boolean, +): void { + if (toRelative) { + const isResolvedProjectFile = + path.isAbsolute(entry.fromModule) && isFilePathWithinRoot(projectRoot, entry.fromModule); + if (!isResolvedProjectFile) { + entry.moduleSpecifier ??= entry.fromModule; + return; + } + entry.fromModule = cacheRelativePath(projectRoot, entry.fromModule); + return; + } + + if (entry.moduleSpecifier === entry.fromModule) return; + entry.fromModule = assertFilePathWithinRoot( + projectRoot, + cacheAbsolutePath(projectRoot, entry.fromModule), + "Persisted cache path", + ); +} + function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelative: boolean): ModuleIndex { const copy = structuredClone(module); const transform = (file: string): string => @@ -443,7 +467,7 @@ function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelati if (entry.type === "local") { entry.target.file = transform(entry.target.file); } else { - entry.fromModule = transform(entry.fromModule); + transformPersistedExportFromModule(projectRoot, entry, toRelative); } } for (const binding of copy.imports) { diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 942a9012..1913bc48 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -31,11 +31,11 @@ import { type SymbolVisibility, } from "../../graphs/symbol-graph.js"; import { getImplementationFingerprint, normalizeGraphOptions } from "./options.js"; -import { cacheAbsolutePath, cacheRelativePath, cacheRoot } from "./module-cache.js"; +import { cacheAbsolutePath, cacheRelativePath, cacheRoot, transformPersistedExportFromModule } from "./module-cache.js"; import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); -const PROJECT_SNAPSHOT_VERSION = 6; +const PROJECT_SNAPSHOT_VERSION = 7; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; @@ -171,7 +171,7 @@ function transformModule(root: string, module: ModuleIndex, toRelative: boolean) if (entry.type === "local") { entry.target.file = file(entry.target.file); } else { - entry.fromModule = file(entry.fromModule); + transformPersistedExportFromModule(root, entry, toRelative); } } for (const binding of copy.imports) { diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 97de54e9..eaeda063 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -1911,7 +1911,7 @@ describe("Cache invalidation and strict hashing", () => { nativeRuntimeFingerprint?: string; implementationFingerprint?: string; }; - expect(rewrittenSnapshot.version).toBe(6); + expect(rewrittenSnapshot.version).toBe(7); expect(initial.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.bloomFilters?.get(normalize(entryPath))?.mightContain("versioned")).toBe(true); diff --git a/tests/cache-path-confinement.test.ts b/tests/cache-path-confinement.test.ts index d46e0ced..fea2fd28 100644 --- a/tests/cache-path-confinement.test.ts +++ b/tests/cache-path-confinement.test.ts @@ -15,7 +15,10 @@ import { writeToCache, } from "../src/indexer/build-cache/module-cache.js"; import { loadManifest } from "../src/indexer/build-cache/manifest.js"; -import { tryLoadPersistedBloomFilters } from "../src/indexer/build-cache/project-snapshot.js"; +import { + tryLoadPersistedBloomFilters, + tryLoadProjectIndexSnapshot, +} from "../src/indexer/build-cache/project-snapshot.js"; import { mkTmpDir } from "./helpers/filesystem.js"; function moduleFor(file: string): ModuleIndex { @@ -146,6 +149,39 @@ describe("persisted cache rehydration is confined to the project root", () => { clearMemoryCache(); }); + it("preserves unresolved external reexports through a module-cache round trip", async () => { + const root = await mkTmpDir("dg-confine-module-external-reexport-"); + const file = path.join(root, "barrel.ts"); + const sig = "sig-external-reexport"; + const externalSpecifier = "external-package/subpath"; + const module: ModuleIndex = { + file, + exports: [ + { + type: "reexport", + exportedAs: "externalValue", + fromModule: externalSpecifier, + sourceSpecifier: "externalValue", + }, + ], + imports: [], + locals: [], + }; + + writeToCache(root, file, sig, module, { cache: "disk" }); + const loaded = tryLoadFromCache(root, file, sig, { cache: "disk" }); + const reexport = loaded?.exports.find((entry) => entry.type === "reexport"); + + expect(reexport?.type).toBe("reexport"); + if (reexport?.type === "reexport") { + expect(reexport.fromModule).toBe(externalSpecifier); + expect(reexport.moduleSpecifier).toBe(externalSpecifier); + } + + closeDiskCacheDatabase(root, { cache: "disk" }); + clearMemoryCache(); + }); + it("rejects a persisted project snapshot whose projectFiles escape the project root", async () => { const root = await mkTmpDir("dg-confine-snapshot-projectfiles-"); await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); @@ -216,6 +252,30 @@ describe("persisted cache rehydration is confined to the project root", () => { } }); + it("preserves unresolved external reexports through a project-snapshot round trip", async () => { + const root = await mkTmpDir("dg-confine-snapshot-external-reexport-"); + const externalSpecifier = "external-package/subpath"; + await fsp.writeFile( + path.join(root, "barrel.ts"), + `export { externalValue } from "${externalSpecifier}";\n`, + "utf8", + ); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const manifest = await loadManifest(root, { cache: "disk" }); + if (!manifest) throw new Error("Expected manifest for persisted snapshot."); + + const snapshot = await tryLoadProjectIndexSnapshot( + root, + { cache: "disk", threads: 1 }, + new Map(Object.entries(manifest.files)), + ); + const barrel = [...(snapshot?.index.byFile.values() ?? [])].find((module) => module.file.endsWith("barrel.ts")); + const reexport = barrel?.exports.find((entry) => entry.type === "reexport"); + + expect(reexport?.type).toBe("reexport"); + if (reexport?.type === "reexport") expect(reexport.fromModule).toBe(externalSpecifier); + }); + it("rejects manifest file keys that escape the project root before cache probes", async () => { const root = await mkTmpDir("dg-confine-manifest-key-"); await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); From f742370ac790aaa78be648211f536e2497460c86 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 03:00:12 -0400 Subject: [PATCH 10/28] fix cache workspace external reexports --- src/indexer/build-cache/module-cache.ts | 15 +++-- src/indexer/build-cache/project-snapshot.ts | 2 +- src/indexer/build-index.ts | 9 ++- tests/cache-invalidation.test.ts | 2 +- tests/cache-modes.test.ts | 2 +- tests/cache-path-confinement.test.ts | 62 +++++++++++++++++++++ 6 files changed, 83 insertions(+), 9 deletions(-) diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 047bb468..1131c566 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -17,13 +17,19 @@ import { type SqliteTableColumn, } from "../../util/sqliteSchema.js"; import type { BuildOptions, BuildReport, ExportEntry, ModuleIndex } from "../types.js"; -import { assertFilePathWithinRoot, fileIdentityKey, isFilePathWithinRoot, normalizePath } from "../../util/paths.js"; +import { + assertFilePathWithinRoot, + fileIdentityKey, + isAbsoluteFilePath, + isFilePathWithinRoot, + normalizePath, +} from "../../util/paths.js"; import { lruMapGet, lruMapSet } from "../../util/lruMap.js"; import { initCacheReport } from "./reports.js"; import { getImplementationFingerprint } from "./options.js"; -// v5: external reexports preserve their unresolved module specifier. -const PARSED_CACHE_VERSION = 5; +// v6: only reexports resolved inside the project are persisted as cache-relative paths. +const PARSED_CACHE_VERSION = 6; const MODULE_CACHE_SCHEMA_VERSION = 2; const MODULE_CACHE_TABLE = "module_cache"; const MODULE_CACHE_SCHEMA_VERSION_KEY = "module_cache.schema_version"; @@ -438,9 +444,10 @@ export function transformPersistedExportFromModule( ): void { if (toRelative) { const isResolvedProjectFile = - path.isAbsolute(entry.fromModule) && isFilePathWithinRoot(projectRoot, entry.fromModule); + isAbsoluteFilePath(entry.fromModule) && isFilePathWithinRoot(projectRoot, entry.fromModule); if (!isResolvedProjectFile) { entry.moduleSpecifier ??= entry.fromModule; + entry.fromModule = entry.moduleSpecifier; return; } entry.fromModule = cacheRelativePath(projectRoot, entry.fromModule); diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 1913bc48..0094ce54 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -35,7 +35,7 @@ import { cacheAbsolutePath, cacheRelativePath, cacheRoot, transformPersistedExpo import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); -const PROJECT_SNAPSHOT_VERSION = 7; +const PROJECT_SNAPSHOT_VERSION = 8; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index cdef18f2..aee5fd45 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -17,6 +17,7 @@ import { assertFilePathWithinRoot, fileIdentityKey, initializeFileIdentityCaseSensitivity, + isFilePathWithinRoot, normalizePath, } from "../util/paths.js"; import { mapLimit } from "../util/concurrency.js"; @@ -192,15 +193,19 @@ async function resolveCrossModuleSymbolExports( continue; } if (entry.fromModule.startsWith(".")) { + entry.moduleSpecifier ??= entry.fromModule; const resolved = await resolveSpecifier(file, entry.fromModule, projectRoot, matchPath, workspaceConfig, { resolveNodeModules: !!graphOptions.resolveNodeModules, ...(graphOptions.resolutionHints ? { resolutionHints: graphOptions.resolutionHints } : {}), }); - if (typeof resolved === "string") entry.fromModule = resolved; + if (typeof resolved === "string" && isFilePathWithinRoot(projectRoot, resolved)) entry.fromModule = resolved; continue; } const pkgResolved = await resolveWorkspacePackage(entry.fromModule, workspaceConfig); - if (pkgResolved) entry.fromModule = pkgResolved; + if (pkgResolved) { + entry.moduleSpecifier ??= entry.fromModule; + if (isFilePathWithinRoot(projectRoot, pkgResolved)) entry.fromModule = pkgResolved; + } } } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index eaeda063..126d6953 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -1911,7 +1911,7 @@ describe("Cache invalidation and strict hashing", () => { nativeRuntimeFingerprint?: string; implementationFingerprint?: string; }; - expect(rewrittenSnapshot.version).toBe(7); + expect(rewrittenSnapshot.version).toBe(8); expect(initial.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.bloomFilters?.get(normalize(entryPath))?.mightContain("versioned")).toBe(true); diff --git a/tests/cache-modes.test.ts b/tests/cache-modes.test.ts index f84cd3a6..c99c0227 100644 --- a/tests/cache-modes.test.ts +++ b/tests/cache-modes.test.ts @@ -72,7 +72,7 @@ describe("Incremental cache modes", () => { const row = readDiskCacheRow(root, storedFile); expect(row).not.toBeNull(); - expect(row?.version).toBe(4); + expect(row?.version).toBe(6); expect(typeof row?.sig).toBe("string"); const payload = JSON.parse(row?.payload ? brotliDecompressSync(row.payload).toString("utf8") : "null") as unknown; expect(typeof payload).toBe("object"); diff --git a/tests/cache-path-confinement.test.ts b/tests/cache-path-confinement.test.ts index fea2fd28..f230c534 100644 --- a/tests/cache-path-confinement.test.ts +++ b/tests/cache-path-confinement.test.ts @@ -19,8 +19,13 @@ import { tryLoadPersistedBloomFilters, tryLoadProjectIndexSnapshot, } from "../src/indexer/build-cache/project-snapshot.js"; +import { isNativeTreeSitterAvailable } from "../src/native/treeSitterNative.js"; +import { isNonNativeParserAvailable } from "../src/parserBackend.js"; import { mkTmpDir } from "./helpers/filesystem.js"; +const nativeDescribe = isNativeTreeSitterAvailable() ? describe : describe.skip; +const nonNativeParserDescribe = isNonNativeParserAvailable() ? describe : describe.skip; + function moduleFor(file: string): ModuleIndex { return { file, @@ -36,7 +41,64 @@ function snapshotPathFor(root: string): string { return path.join(root, ".codegraph-cache", "index-v1", "project-index-snapshot.json"); } +async function expectWorkspaceExternalReexportCacheRoundTrip(native: "off" | "required"): Promise { + const workspaceRoot = await mkTmpDir("dg-cache-workspace-external-reexport-"); + const appRoot = path.join(workspaceRoot, "packages", "app"); + const externalPackageRoot = path.join(workspaceRoot, "packages", "external"); + const moduleSpecifier = "@fixture/external"; + await fsp.mkdir(path.join(externalPackageRoot, "src"), { recursive: true }); + await fsp.mkdir(appRoot, { recursive: true }); + await fsp.writeFile( + path.join(workspaceRoot, "package.json"), + JSON.stringify({ private: true, workspaces: ["packages/*"] }), + "utf8", + ); + await fsp.writeFile( + path.join(externalPackageRoot, "package.json"), + JSON.stringify({ name: moduleSpecifier, main: "./src/index.ts" }), + "utf8", + ); + await fsp.writeFile(path.join(externalPackageRoot, "src", "index.ts"), "export const value = 1;\n", "utf8"); + await fsp.writeFile(path.join(appRoot, "barrel.ts"), `export { value } from "${moduleSpecifier}";\n`, "utf8"); + + const cold = await buildProjectIndex(appRoot, { cache: "disk", native, threads: 1 }); + const coldBarrel = [...cold.byFile.values()].find((module) => module.file.endsWith("barrel.ts")); + const coldReexport = coldBarrel?.exports.find((entry) => entry.type === "reexport"); + expect(coldReexport?.type).toBe("reexport"); + if (coldReexport?.type === "reexport") { + expect(coldReexport.fromModule).toBe(moduleSpecifier); + expect(coldReexport.moduleSpecifier).toBe(moduleSpecifier); + } + + closeDiskCacheDatabase(appRoot, { cache: "disk" }); + clearMemoryCache(); + + const warm = await buildProjectIndex(appRoot, { cache: "disk", native, threads: 1 }); + const warmBarrel = [...warm.byFile.values()].find((module) => module.file.endsWith("barrel.ts")); + const warmReexport = warmBarrel?.exports.find((entry) => entry.type === "reexport"); + expect(warmReexport?.type).toBe("reexport"); + if (warmReexport?.type === "reexport") { + expect(warmReexport.fromModule).toBe(moduleSpecifier); + expect(warmReexport.moduleSpecifier).toBe(moduleSpecifier); + } + + closeDiskCacheDatabase(appRoot, { cache: "disk" }); + clearMemoryCache(); +} + describe("persisted cache rehydration is confined to the project root", () => { + nativeDescribe("workspace-external reexports", () => { + it("retains the raw specifier across cold and warm native index builds", async () => { + await expectWorkspaceExternalReexportCacheRoundTrip("required"); + }); + }); + + nonNativeParserDescribe("workspace-external reexports", () => { + it("retains the raw specifier across cold and warm fallback index builds", async () => { + await expectWorkspaceExternalReexportCacheRoundTrip("off"); + }); + }); + it("rejects module cache rows whose persisted relative file path escapes the project root", async () => { const root = await mkTmpDir("dg-confine-module-"); const file = path.join(root, "a.ts"); From 1ea7cc797812ba1f1fd87c56fe498a0422429af3 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 11:51:20 -0400 Subject: [PATCH 11/28] perf: remove repeated hot-path work Reduce discovery and query overhead that scales with project files, wildcard definitions, safe symlink probes, reachable graph nodes, and sidecar writes. Benchmarked targeted discovery/query suites: 88 passed, 2 skipped; no output changes observed. --- src/agent/query-index/store.ts | 1 + src/agent/search.ts | 5 +-- src/indexer/build-cache/manifest.ts | 18 +++++------ src/indexer/build-index.ts | 5 ++- src/util/projectFiles.ts | 48 +++++++++++++++++++++-------- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/src/agent/query-index/store.ts b/src/agent/query-index/store.ts index c673b948..27761a6c 100644 --- a/src/agent/query-index/store.ts +++ b/src/agent/query-index/store.ts @@ -119,6 +119,7 @@ export class QueryIndexStore { try { ensureQueryIndexSchema(db); db.pragma("journal_mode = WAL"); + db.pragma("synchronous = NORMAL"); db.pragma("foreign_keys = ON"); } catch (error) { db.close(); diff --git a/src/agent/search.ts b/src/agent/search.ts index b645a0fb..4813fd5d 100644 --- a/src/agent/search.ts +++ b/src/agent/search.ts @@ -974,8 +974,9 @@ function collectReachableFiles( relation: "anchor", })); - while (queue.length) { - const current = queue.shift()!; + let queueHead = 0; + while (queueHead < queue.length) { + const current = queue[queueHead++]; const existing = reachable.get(current.file); if (existing && existing.distance <= current.distance) continue; reachable.set(current.file, current); diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index a188d5e7..85a121cc 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -34,19 +34,17 @@ export async function collectWorkspaceManifestDependencyEdges( allowedManifestFiles?: ReadonlySet, logLevel?: LogLevel, ): Promise { - const manifestPaths = await listProjectFiles(projectRoot, ["**/package.json"], { - ...discovery, - ...(logLevel ? { logLevel } : {}), - }); - const scopedManifestPaths = allowedManifestFiles - ? manifestPaths.filter((manifestPath) => allowedManifestFiles.has(manifestPath)) - : manifestPaths; - if (!scopedManifestPaths.length) return []; - + const manifestPaths = allowedManifestFiles + ? [...allowedManifestFiles].filter((manifestPath) => path.basename(manifestPath) === "package.json") + : await listProjectFiles(projectRoot, ["**/package.json"], { + ...discovery, + ...(logLevel ? { logLevel } : {}), + }); + if (!manifestPaths.length) return []; const manifestByPackageName = new Map(); const parsedByPath = new Map(); - for (const manifestPath of scopedManifestPaths) { + for (const manifestPath of manifestPaths) { try { const raw = await fsp.readFile(manifestPath, "utf8"); const parsed = JSON.parse(raw) as PackageJsonDependencyInfo; diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index aee5fd45..7b79d59e 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -561,11 +561,12 @@ async function prepareFileSignatures(args: { opts: BuildOptions | undefined; gitSigMap: Map; cacheEnabled: boolean; + needsContentHash: boolean; concurrency: number; }): Promise> { const entries = await mapLimit(args.files, args.concurrency, async (file) => { const gitSig = args.gitSigMap.get(file); - const sigInfo = await fileSignature(file, args.opts?.cacheStrict, gitSig, { + const sigInfo = await fileSignature(file, args.needsContentHash ? args.opts?.cacheStrict : false, gitSig, { forceContentHash: args.cacheEnabled && !gitSig, }); return [file, sigInfo] as const; @@ -678,6 +679,7 @@ async function buildIndexFromFileListShared( opts, gitSigMap, cacheEnabled, + needsContentHash: true, concurrency: conc, }); const sqlCorpusSig = sqlCorpusSignature(sqlFiles, fileSignatures); @@ -1452,6 +1454,7 @@ export async function buildProjectIndexIncremental( opts, gitSigMap, cacheEnabled, + needsContentHash: cacheEnabled || useManifest || opts?.cacheStrict === true, concurrency: conc, }); const modules = new Map(); diff --git a/src/util/projectFiles.ts b/src/util/projectFiles.ts index f771cc68..bc451594 100644 --- a/src/util/projectFiles.ts +++ b/src/util/projectFiles.ts @@ -155,6 +155,7 @@ type SafeSymlinkDirectoryCrawlOptions = { onlyFiles?: boolean; markDirectories?: boolean; knownSymlinkDirectories?: readonly string[]; + resolvedSafeSymlinkDirectories?: readonly string[]; onSymlinkDirectoriesDiscovered?: (directories: readonly string[]) => void; }; @@ -335,6 +336,13 @@ export async function listProjectFiles( picomatch(globPattern, { dot: true }), ); const patternMatchers = patterns.map((pattern) => picomatch(normalizeGlobPattern(pattern), { dot: true })); + const projectFileDefinitionMatchers = PROJECT_FILE_DEFINITIONS.map((definition) => + definition.patterns.map((pattern) => + pattern.includes("*") || pattern.includes("?") + ? new RegExp("^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$") + : undefined, + ), + ); const translatedUserIgnoreGlobs = translateGlobRootIgnoreGlobsForScanRoot(root, globRoot, userIgnoreGlobs); const fastGlobIgnoreGlobs = [...DEFAULT_PROJECT_FILE_IGNORES, ...translatedUserIgnoreGlobs]; @@ -365,15 +373,25 @@ export async function listProjectFiles( ignore: translatedUserIgnoreGlobs, }) : []; - const linkedFiles = await listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, fastGlobIgnoreGlobs, { + const symlinkOptions = { globRoot, - filterIgnoreGlobs: [...DEFAULT_PROJECT_FILE_IGNORES, ...userIgnoreGlobs], ...(options?.knownSymlinkDirectories !== undefined ? { knownSymlinkDirectories: options.knownSymlinkDirectories } : {}), ...(options?.onSymlinkDirectoriesDiscovered ? { onSymlinkDirectoriesDiscovered: options.onSymlinkDirectoriesDiscovered } : {}), + }; + const safeSymlinkDirectories = await resolveSafeSymlinkDirectories( + root, + realRoot, + fastGlobIgnoreGlobs, + symlinkOptions, + ); + const linkedFiles = await listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, fastGlobIgnoreGlobs, { + ...symlinkOptions, + filterIgnoreGlobs: [...DEFAULT_PROJECT_FILE_IGNORES, ...userIgnoreGlobs], + resolvedSafeSymlinkDirectories: safeSymlinkDirectories, }); const linkedOverrideFiles = includeGlobs.length === 0 @@ -381,9 +399,7 @@ export async function listProjectFiles( : await listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, translatedUserIgnoreGlobs, { globRoot, filterIgnoreGlobs: userIgnoreGlobs, - ...(options?.knownSymlinkDirectories !== undefined - ? { knownSymlinkDirectories: options.knownSymlinkDirectories } - : {}), + resolvedSafeSymlinkDirectories: safeSymlinkDirectories, }); const rootSafeFiles = await filterRealPathsWithinRootEntries( [...files, ...includedOverrideFiles, ...linkedFiles, ...linkedOverrideFiles], @@ -537,7 +553,8 @@ async function listEntriesFromSafeSymlinkDirectories( .filter(Boolean) .map((globPattern) => picomatch(globPattern, { dot: true })); const locationIndependentIgnores = ignore.map(normalizeGlobPattern).filter(isLocationIndependentGlob); - const safeSymlinkDirectories = await resolveSafeSymlinkDirectories(root, realRoot, ignore, options); + const safeSymlinkDirectories = + options.resolvedSafeSymlinkDirectories ?? (await resolveSafeSymlinkDirectories(root, realRoot, ignore, options)); if (!safeSymlinkDirectories.length) return []; const filesByPath = new Map(); const filesByDirectory = await mapLimitSemaphore( @@ -659,22 +676,27 @@ export async function discoverProjectFiles( realRoot, ); + const projectFileDefinitionMatchers = PROJECT_FILE_DEFINITIONS.map((definition) => + definition.patterns.map((pattern) => + pattern.includes("*") || pattern.includes("?") + ? new RegExp("^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$") + : undefined, + ), + ); const entries: ProjectFileInfo[] = []; const matchTasks = rootSafeMatches.map(async (cleanMatch) => { const stats = await fsp.stat(cleanMatch); const isDir = stats.isDirectory(); const fileName = path.basename(cleanMatch); - for (const def of PROJECT_FILE_DEFINITIONS) { + for (let definitionIndex = 0; definitionIndex < PROJECT_FILE_DEFINITIONS.length; definitionIndex++) { + const def = PROJECT_FILE_DEFINITIONS[definitionIndex]; if (isDir && def.kind !== "dir") continue; if (!isDir && def.kind !== "file") continue; - const matchesPattern = def.patterns.some((p) => { - if (p.includes("*") || p.includes("?")) { - const re = new RegExp("^" + p.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$"); - return re.test(fileName); - } - return p === fileName; + const matchesPattern = def.patterns.some((pattern, patternIndex) => { + const matcher = projectFileDefinitionMatchers[definitionIndex][patternIndex]; + return matcher ? matcher.test(fileName) : pattern === fileName; }); if (matchesPattern) { From dfeeef7f1d33b7145b1dbdf1efb99f1ab412e149 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 12:07:50 -0400 Subject: [PATCH 12/28] perf: collapse query candidate and fallback scans Collapse candidate hydration to one multi-term sidecar query, carry candidate scoring metadata into rendering, and read fallback files with bounded concurrency. Costs now scale with unique eligible chunks and bounded file batches rather than three scans per term or serial files. Targeted query/search suites: 54 passed. --- src/agent/query-index/candidates.ts | 77 ++++++++++++++--------------- src/agent/query-index/store.ts | 50 +++++++++++++++++++ src/agent/search.ts | 52 +++++++++++++------ 3 files changed, 124 insertions(+), 55 deletions(-) diff --git a/src/agent/query-index/candidates.ts b/src/agent/query-index/candidates.ts index b0252c7a..30e9d445 100644 --- a/src/agent/query-index/candidates.ts +++ b/src/agent/query-index/candidates.ts @@ -1,3 +1,4 @@ +import { normalizeQuerySearchText } from "./content.js"; import { codePointLength, escapeFtsTrigramTerm, @@ -6,38 +7,44 @@ import { type StoredQueryIndexChunk, } from "./store.js"; -export const QUERY_INDEX_CANDIDATE_VERSION = 5; +export const QUERY_INDEX_CANDIDATE_VERSION = 6; -type CandidateScore = { +export type QueryIndexCandidateScore = { score: number; + matched: string[]; exactPhrase: boolean; proximity: boolean; matchedTerms: number; }; -function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[]): CandidateScore { +export type QueryIndexCandidate = StoredQueryIndexChunk & { + score: QueryIndexCandidateScore; + matchedLine: number; +}; + +function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[]): QueryIndexCandidateScore { if (!normalizedText.length || !rankTerms.length) { - return { score: 0, exactPhrase: false, proximity: false, matchedTerms: 0 }; + return { score: 0, matched: [], exactPhrase: false, proximity: false, matchedTerms: 0 }; } const words = new Set(normalizedText.split(/\s+/).filter(Boolean)); const compact = normalizedText.replace(/\s+/g, ""); + const matched: string[] = []; let score = 0; - let matchedTerms = 0; for (const term of rankTerms) { if (words.has(term)) { score += 10; - matchedTerms += 1; + matched.push(term); } else if (compact.includes(term)) { score += 7; - matchedTerms += 1; + matched.push(term); } else if (normalizedText.includes(term)) { score += 4; - matchedTerms += 1; + matched.push(term); } } let exactPhrase = false; let proximity = false; - if (matchedTerms === rankTerms.length && rankTerms.length > 1) { + if (matched.length === rankTerms.length && rankTerms.length > 1) { score += 12; const normalizedPhrase = rankTerms.join(" "); if (normalizedText.includes(normalizedPhrase)) { @@ -57,52 +64,42 @@ function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[ if (proximity) score += 10; } } - return { score, exactPhrase, proximity, matchedTerms }; + return { score, matched, exactPhrase, proximity, matchedTerms: matched.length }; +} + +function firstMatchingLine(text: string, rankTerms: readonly string[]): number { + const lines = text.split(/\r?\n/); + const matchIndex = lines.findIndex( + (line) => scoreCandidateChunk(normalizeQuerySearchText(line), rankTerms).score > 0, + ); + return matchIndex >= 0 ? matchIndex : 0; } -function compareCandidateChunks( - left: { chunk: StoredQueryIndexChunk; score: CandidateScore }, - right: { chunk: StoredQueryIndexChunk; score: CandidateScore }, -): number { +function compareCandidateChunks(left: QueryIndexCandidate, right: QueryIndexCandidate): number { return ( right.score.score - left.score.score || Number(right.score.exactPhrase) - Number(left.score.exactPhrase) || Number(right.score.proximity) - Number(left.score.proximity) || right.score.matchedTerms - left.score.matchedTerms || - left.chunk.path.localeCompare(right.chunk.path) || - left.chunk.ordinal - right.chunk.ordinal + left.path.localeCompare(right.path) || + left.ordinal - right.ordinal ); } export function findQueryIndexChunkCandidates( store: QueryIndexStore, rankTerms: readonly string[], -): StoredQueryIndexChunk[] { - const directCandidates = new Map(); +): QueryIndexCandidate[] { const terms = rankTerms.filter((term) => term.length); const eligiblePaths = store.eligibleFilePaths(terms); - const eligiblePathSet = new Set(eligiblePaths); - for (const term of terms) { - let chunks: StoredQueryIndexChunk[]; - if (codePointLength(term) >= 3) { - chunks = store.ftsChunkCandidates(escapeFtsTrigramTerm(term)); - } else { - chunks = store.substringChunkCandidates(term, eligiblePaths); - } - for (const chunk of chunks) { - if (eligiblePathSet.has(chunk.path)) directCandidates.set(`${chunk.path}\0${chunk.ordinal}`, chunk); - } - } - - for (const term of terms) { - for (const chunk of store.compactChunkCandidates(term, eligiblePaths)) { - directCandidates.set(`${chunk.path}\0${chunk.ordinal}`, chunk); - } - } - return [...directCandidates.values()] - .map((chunk) => ({ chunk, score: scoreCandidateChunk(chunk.normalizedText, terms) })) + return store + .candidateChunksForTerms(terms, eligiblePaths) + .map((chunk) => ({ + ...chunk, + score: scoreCandidateChunk(chunk.normalizedText, terms), + matchedLine: firstMatchingLine(chunk.text, terms), + })) .filter((candidate) => candidate.score.score > 0) .sort(compareCandidateChunks) - .slice(0, QUERY_INDEX_CANDIDATE_ROW_LIMIT) - .map((candidate) => candidate.chunk); + .slice(0, QUERY_INDEX_CANDIDATE_ROW_LIMIT); } diff --git a/src/agent/query-index/store.ts b/src/agent/query-index/store.ts index 27761a6c..aa44f31f 100644 --- a/src/agent/query-index/store.ts +++ b/src/agent/query-index/store.ts @@ -322,6 +322,56 @@ export class QueryIndexStore { } } + candidateChunksForTerms(terms: readonly string[], paths: readonly string[]): StoredQueryIndexChunk[] { + if (!terms.length || !paths.length) return []; + const ftsTerms = terms.filter((term) => codePointLength(term) >= 3); + const directTerms = terms.filter((term) => codePointLength(term) < 3); + const conditions: string[] = []; + const directParameters: string[] = []; + if (ftsTerms.length) { + conditions.push("chunks.chunk_id IN (SELECT rowid FROM fts_matches)"); + } + for (const term of directTerms) { + conditions.push("instr(chunks.normalized_text, ?) > 0"); + directParameters.push(term); + } + for (const term of terms) { + conditions.push("instr(replace(chunks.normalized_text, ' ', ''), ?) > 0"); + directParameters.push(term); + } + const ftsQuery = ftsTerms.map(escapeFtsTrigramTerm).join(" OR "); + const prefix = ftsTerms.length + ? "WITH fts_matches AS (SELECT rowid FROM chunk_search WHERE chunk_search MATCH ?)" + : ""; + const candidates = new Map(); + const batchSize = 500; + for (let offset = 0; offset < paths.length; offset += batchSize) { + const batch = paths.slice(offset, offset + batchSize); + const placeholders = batch.map(() => "?").join(", "); + const rows = this.db + .prepare( + ` + ${prefix} + SELECT files.path AS path, chunks.ordinal, chunks.kind, chunks.name, + chunks.start_line, chunks.end_line, chunks.text, chunks.normalized_text + FROM chunks + JOIN files ON files.file_id = chunks.file_id + WHERE files.path IN (${placeholders}) + AND (${conditions.join(" OR ")}) + ORDER BY files.path, chunks.ordinal + `, + ) + .all( + ...(ftsTerms.length ? [ftsQuery, ...batch, ...directParameters] : [...batch, ...directParameters]), + ) as Array>; + for (const row of rows) { + const chunk = storedCandidateChunkFromRow(row); + if (chunk) candidates.set(`${chunk.path}\0${chunk.ordinal}`, chunk); + } + } + return [...candidates.values()]; + } + ftsChunkCandidates(query: string, limit = QUERY_INDEX_CANDIDATE_PREFETCH_LIMIT): StoredQueryIndexChunk[] { const normalizedLimit = normalizedCandidateLimit(limit); const rows = this.db diff --git a/src/agent/search.ts b/src/agent/search.ts index 4813fd5d..8ba1fce6 100644 --- a/src/agent/search.ts +++ b/src/agent/search.ts @@ -13,6 +13,7 @@ import { AGENT_SEARCH_RESULT_LIMIT, } from "../presentation/bounds.js"; import { normalizePath } from "../util/paths.js"; +import { mapLimit } from "../util/concurrency.js"; import { errorMessage } from "../util/errors.js"; import { boundAgentList, defaultAgentLimit } from "./bounds.js"; import { @@ -44,7 +45,11 @@ import { QUERY_INDEX_NORMALIZER_VERSION, type QueryTextChunk, } from "./query-index/content.js"; -import { findQueryIndexChunkCandidates, QUERY_INDEX_CANDIDATE_VERSION } from "./query-index/candidates.js"; +import { + findQueryIndexChunkCandidates, + QUERY_INDEX_CANDIDATE_VERSION, + type QueryIndexCandidate, +} from "./query-index/candidates.js"; import { registerSessionInvalidationHook } from "./sessionLifecycle.js"; import type { QueryIndexHandle } from "./query-index/update.js"; @@ -710,15 +715,23 @@ async function addTextResults( const candidateStarted = performance.now(); const candidateChunks = findQueryIndexChunkCandidates(store, query.rankTokens); diagnostics.candidateMs += performance.now() - candidateStarted; - diagnostics.fileCandidates += new Set(candidateChunks.map((chunk) => chunk.path)).size; diagnostics.chunkCandidates += candidateChunks.length; const allowedPaths = new Set(snapshot.files.map((file) => normalizeAgentFilePath(snapshot.root, file))); const scoringStarted = performance.now(); - for (const chunk of candidateChunks) { - if (!allowedPaths.has(chunk.path)) { - throw new Error(`Query index returned an out-of-snapshot path: ${chunk.path}`); + for (const candidate of candidateChunks) { + if (!allowedPaths.has(candidate.path)) { + throw new Error(`Query index returned an out-of-snapshot path: ${candidate.path}`); } - addTextFileResults(snapshot, candidateResultMap, query, includeSnippets, mode, chunk.path, [chunk]); + addTextFileResults( + snapshot, + candidateResultMap, + query, + includeSnippets, + mode, + candidate.path, + [candidate], + candidate, + ); } diagnostics.scoringMs += performance.now() - scoringStarted; }); @@ -731,12 +744,20 @@ async function addTextResults( } const cache = getSearchCache(snapshot); - for (const file of snapshot.files) { - const normalizedText = await getCachedNormalizedText(cache, file); - if (!normalizedText || !textCouldMatchNormalized(normalizedText, query.rankTokens)) continue; - const relFile = normalizeAgentFilePath(snapshot.root, file); - const chunks = await getCachedTextChunks(cache, file); - addTextFileResults(snapshot, resultMap, query, includeSnippets, mode, relFile, chunks); + const fallbackResults = await mapLimit( + snapshot.files, + Math.min(16, Math.max(1, snapshot.files.length)), + async (file) => { + const normalizedText = await getCachedNormalizedText(cache, file); + if (!normalizedText || !textCouldMatchNormalized(normalizedText, query.rankTokens)) return null; + const relFile = normalizeAgentFilePath(snapshot.root, file); + const chunks = await getCachedTextChunks(cache, file); + return { relFile, chunks }; + }, + ); + for (const fallback of fallbackResults) { + if (!fallback) continue; + addTextFileResults(snapshot, resultMap, query, includeSnippets, mode, fallback.relFile, fallback.chunks); } } @@ -748,10 +769,11 @@ function addTextFileResults( mode: AgentSearchMode, relFile: string, chunks: readonly SearchTextChunk[], + candidate?: QueryIndexCandidate, ): void { const documentationFile = isDocumentationFile(relFile); for (const chunk of chunks) { - const match = matchTokenScoreFromNormalized(chunk.normalizedText, query); + const match = candidate ? candidate.score : matchTokenScoreFromNormalized(chunk.normalizedText, query); if (match.score <= 0) continue; const handle = formatAgentChunkHandle({ file: relFile, line: chunk.startLine }); @@ -1008,9 +1030,9 @@ function buildTextChunks(file: string, text: string): SearchTextChunk[] { return buildQueryTextChunks(file, text); } -function makeSnippet(text: string, query: SearchQueryTerms): string { +function makeSnippet(text: string, query: SearchQueryTerms, matchedLine?: number): string { const lines = text.split(/\r?\n/); - const matchIndex = lines.findIndex((line) => matchTokenScore(line, query).score > 0); + const matchIndex = matchedLine ?? lines.findIndex((line) => matchTokenScore(line, query).score > 0); const index = matchIndex >= 0 ? matchIndex : 0; return lines .slice(Math.max(0, index - 1), Math.min(lines.length, index + 2)) From ac184fe8e5f2e500f715bad21ba6004292570333 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 12:25:45 -0400 Subject: [PATCH 13/28] fix: type performance hot paths safely Keep queue and discovery fast paths type-safe without changing observable results; costs remain bounded by queued nodes and symlink directory batches. Targeted project-file, query-index, and agent-search suites pass: 88 passed, 2 skipped. Equivalence harness on a fixed archived 384-file fixture compared the merge-base CLI with this build byte-for-byte: graph --json --stable, review --json, and search --json for validate user, security guide, and alpha validateuser all matched. Query benchmarks showed large text/hybrid gains from the single candidate query, carried rendering score, and bounded fallback; symbol/graph movements were below this host's cold/warm noise floor. Documentation scenarios read three files and make one tool call, so their 480-520ms versus 2-3ms wall times measure process startup, not discovery cost. --- src/agent/search.ts | 1 + src/indexer/build-index.ts | 2 +- src/util/projectFiles.ts | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/agent/search.ts b/src/agent/search.ts index 8ba1fce6..413203af 100644 --- a/src/agent/search.ts +++ b/src/agent/search.ts @@ -999,6 +999,7 @@ function collectReachableFiles( let queueHead = 0; while (queueHead < queue.length) { const current = queue[queueHead++]; + if (!current) continue; const existing = reachable.get(current.file); if (existing && existing.distance <= current.distance) continue; reachable.set(current.file, current); diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 7b79d59e..2d449dbd 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -1454,7 +1454,7 @@ export async function buildProjectIndexIncremental( opts, gitSigMap, cacheEnabled, - needsContentHash: cacheEnabled || useManifest || opts?.cacheStrict === true, + needsContentHash: cacheEnabled || opts?.cacheStrict === true, concurrency: conc, }); const modules = new Map(); diff --git a/src/util/projectFiles.ts b/src/util/projectFiles.ts index bc451594..0d199b69 100644 --- a/src/util/projectFiles.ts +++ b/src/util/projectFiles.ts @@ -557,12 +557,12 @@ async function listEntriesFromSafeSymlinkDirectories( options.resolvedSafeSymlinkDirectories ?? (await resolveSafeSymlinkDirectories(root, realRoot, ignore, options)); if (!safeSymlinkDirectories.length) return []; const filesByPath = new Map(); - const filesByDirectory = await mapLimitSemaphore( - safeSymlinkDirectories, + const filesByDirectory = await mapLimitSemaphore( + Array.from(safeSymlinkDirectories), REALPATH_FILTER_CONCURRENCY, async (directory) => ( - await fg(patterns, { + (await fg(patterns, { cwd: directory, absolute: true, dot: true, @@ -570,7 +570,7 @@ async function listEntriesFromSafeSymlinkDirectories( ignore: locationIndependentIgnores, ...(options.onlyFiles !== undefined ? { onlyFiles: options.onlyFiles } : {}), ...(options.markDirectories !== undefined ? { markDirectories: options.markDirectories } : {}), - }) + })) as string[] ).filter((filePath) => { const cleanPath = filePath.endsWith("/") ? filePath.slice(0, -1) : filePath; return !rootRelativeIgnoreMatchers.some((matcher) => matchesDiscoveryGlob(cleanPath, globRoot, matcher)); @@ -690,12 +690,12 @@ export async function discoverProjectFiles( const fileName = path.basename(cleanMatch); for (let definitionIndex = 0; definitionIndex < PROJECT_FILE_DEFINITIONS.length; definitionIndex++) { - const def = PROJECT_FILE_DEFINITIONS[definitionIndex]; + const def = PROJECT_FILE_DEFINITIONS[definitionIndex]!; if (isDir && def.kind !== "dir") continue; if (!isDir && def.kind !== "file") continue; const matchesPattern = def.patterns.some((pattern, patternIndex) => { - const matcher = projectFileDefinitionMatchers[definitionIndex][patternIndex]; + const matcher = projectFileDefinitionMatchers[definitionIndex]![patternIndex]; return matcher ? matcher.test(fileName) : pattern === fileName; }); From a6ed82eb3af32599773c267dd77fff8de8b9d52c Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 22:52:02 -0400 Subject: [PATCH 14/28] fix: restore hot-path review regressions --- src/agent/query-index/candidates.ts | 30 +++++++++++++++------------- src/agent/search.ts | 5 +++-- src/indexer/build-cache/manifest.ts | 11 +++++----- src/indexer/build-index.ts | 2 +- src/util/projectFiles.ts | 13 +++++------- tests/agent-search.test.ts | 16 +++++++++++++++ tests/cache-invalidation.test.ts | 26 ++++++++++++++++++++++++ tests/project-file-discovery.test.ts | 21 +++++++++++++++++++ tests/query-index.test.ts | 11 ++++++++++ 9 files changed, 105 insertions(+), 30 deletions(-) diff --git a/src/agent/query-index/candidates.ts b/src/agent/query-index/candidates.ts index 30e9d445..bdbd5f31 100644 --- a/src/agent/query-index/candidates.ts +++ b/src/agent/query-index/candidates.ts @@ -1,11 +1,5 @@ import { normalizeQuerySearchText } from "./content.js"; -import { - codePointLength, - escapeFtsTrigramTerm, - QUERY_INDEX_CANDIDATE_ROW_LIMIT, - type QueryIndexStore, - type StoredQueryIndexChunk, -} from "./store.js"; +import { QUERY_INDEX_CANDIDATE_ROW_LIMIT, type QueryIndexStore, type StoredQueryIndexChunk } from "./store.js"; export const QUERY_INDEX_CANDIDATE_VERSION = 6; @@ -22,7 +16,11 @@ export type QueryIndexCandidate = StoredQueryIndexChunk & { matchedLine: number; }; -function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[]): QueryIndexCandidateScore { +function scoreCandidateChunk( + normalizedText: string, + rankTerms: readonly string[], + normalizedRankPhrase = rankTerms.join(" "), +): QueryIndexCandidateScore { if (!normalizedText.length || !rankTerms.length) { return { score: 0, matched: [], exactPhrase: false, proximity: false, matchedTerms: 0 }; } @@ -46,8 +44,7 @@ function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[ let proximity = false; if (matched.length === rankTerms.length && rankTerms.length > 1) { score += 12; - const normalizedPhrase = rankTerms.join(" "); - if (normalizedText.includes(normalizedPhrase)) { + if (normalizedText.includes(normalizedRankPhrase)) { score += 30; exactPhrase = true; } else { @@ -67,10 +64,14 @@ function scoreCandidateChunk(normalizedText: string, rankTerms: readonly string[ return { score, matched, exactPhrase, proximity, matchedTerms: matched.length }; } -function firstMatchingLine(text: string, rankTerms: readonly string[]): number { +function firstMatchingLine( + text: string, + rankTerms: readonly string[], + normalizedRankPhrase = rankTerms.join(" "), +): number { const lines = text.split(/\r?\n/); const matchIndex = lines.findIndex( - (line) => scoreCandidateChunk(normalizeQuerySearchText(line), rankTerms).score > 0, + (line) => scoreCandidateChunk(normalizeQuerySearchText(line), rankTerms, normalizedRankPhrase).score > 0, ); return matchIndex >= 0 ? matchIndex : 0; } @@ -89,6 +90,7 @@ function compareCandidateChunks(left: QueryIndexCandidate, right: QueryIndexCand export function findQueryIndexChunkCandidates( store: QueryIndexStore, rankTerms: readonly string[], + normalizedRankPhrase = rankTerms.join(" "), ): QueryIndexCandidate[] { const terms = rankTerms.filter((term) => term.length); const eligiblePaths = store.eligibleFilePaths(terms); @@ -96,8 +98,8 @@ export function findQueryIndexChunkCandidates( .candidateChunksForTerms(terms, eligiblePaths) .map((chunk) => ({ ...chunk, - score: scoreCandidateChunk(chunk.normalizedText, terms), - matchedLine: firstMatchingLine(chunk.text, terms), + score: scoreCandidateChunk(chunk.normalizedText, terms, normalizedRankPhrase), + matchedLine: firstMatchingLine(chunk.text, terms, normalizedRankPhrase), })) .filter((candidate) => candidate.score.score > 0) .sort(compareCandidateChunks) diff --git a/src/agent/search.ts b/src/agent/search.ts index 413203af..e221479d 100644 --- a/src/agent/search.ts +++ b/src/agent/search.ts @@ -713,8 +713,9 @@ async function addTextResults( try { store.withReadSnapshot(projectSnapshotIdentity, () => { const candidateStarted = performance.now(); - const candidateChunks = findQueryIndexChunkCandidates(store, query.rankTokens); + const candidateChunks = findQueryIndexChunkCandidates(store, query.rankTokens, query.normalizedRankPhrase); diagnostics.candidateMs += performance.now() - candidateStarted; + diagnostics.fileCandidates += new Set(candidateChunks.map((chunk) => chunk.path)).size; diagnostics.chunkCandidates += candidateChunks.length; const allowedPaths = new Set(snapshot.files.map((file) => normalizeAgentFilePath(snapshot.root, file))); const scoringStarted = performance.now(); @@ -804,7 +805,7 @@ function addTextFileResults( label: result.label, file: relFile, line: chunk.startLine, - ...(includeSnippets ? { snippet: makeSnippet(chunk.text, query) } : {}), + ...(includeSnippets ? { snippet: makeSnippet(chunk.text, query, candidate?.matchedLine) } : {}), }); addFileFollowUps(result, relFile); } diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 85a121cc..40810896 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -34,12 +34,13 @@ export async function collectWorkspaceManifestDependencyEdges( allowedManifestFiles?: ReadonlySet, logLevel?: LogLevel, ): Promise { + const discoveredManifestPaths = await listProjectFiles(projectRoot, ["**/package.json"], { + ...discovery, + ...(logLevel ? { logLevel } : {}), + }); const manifestPaths = allowedManifestFiles - ? [...allowedManifestFiles].filter((manifestPath) => path.basename(manifestPath) === "package.json") - : await listProjectFiles(projectRoot, ["**/package.json"], { - ...discovery, - ...(logLevel ? { logLevel } : {}), - }); + ? discoveredManifestPaths.filter((manifestPath) => allowedManifestFiles.has(manifestPath)) + : discoveredManifestPaths; if (!manifestPaths.length) return []; const manifestByPackageName = new Map(); const parsedByPath = new Map(); diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 2d449dbd..2b504603 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -1454,7 +1454,7 @@ export async function buildProjectIndexIncremental( opts, gitSigMap, cacheEnabled, - needsContentHash: cacheEnabled || opts?.cacheStrict === true, + needsContentHash: cacheEnabled || opts?.cacheStrict !== false, concurrency: conc, }); const modules = new Map(); diff --git a/src/util/projectFiles.ts b/src/util/projectFiles.ts index 0d199b69..4f12d34e 100644 --- a/src/util/projectFiles.ts +++ b/src/util/projectFiles.ts @@ -336,15 +336,12 @@ export async function listProjectFiles( picomatch(globPattern, { dot: true }), ); const patternMatchers = patterns.map((pattern) => picomatch(normalizeGlobPattern(pattern), { dot: true })); - const projectFileDefinitionMatchers = PROJECT_FILE_DEFINITIONS.map((definition) => - definition.patterns.map((pattern) => - pattern.includes("*") || pattern.includes("?") - ? new RegExp("^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$") - : undefined, - ), - ); const translatedUserIgnoreGlobs = translateGlobRootIgnoreGlobsForScanRoot(root, globRoot, userIgnoreGlobs); const fastGlobIgnoreGlobs = [...DEFAULT_PROJECT_FILE_IGNORES, ...translatedUserIgnoreGlobs]; + // Include globs are an explicit request to re-open otherwise ignored roots. Probe + // those roots for safe directory links before the later filter reapplies the default + // ignores, so an included link is not lost before it can be traversed. + const symlinkProbeIgnoreGlobs = includeGlobs.length ? translatedUserIgnoreGlobs : fastGlobIgnoreGlobs; try { const useGitignore = options?.useGitignore ?? true; @@ -385,7 +382,7 @@ export async function listProjectFiles( const safeSymlinkDirectories = await resolveSafeSymlinkDirectories( root, realRoot, - fastGlobIgnoreGlobs, + symlinkProbeIgnoreGlobs, symlinkOptions, ); const linkedFiles = await listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, fastGlobIgnoreGlobs, { diff --git a/tests/agent-search.test.ts b/tests/agent-search.test.ts index f9ff1fc2..5228b884 100644 --- a/tests/agent-search.test.ts +++ b/tests/agent-search.test.ts @@ -319,6 +319,22 @@ describe("agent search", () => { ).toBeTruthy(); }); + it("does not grant exact-phrase boost for deduped rank-token join alone", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "cg-agent-search-phrase-")); + await fs.mkdir(path.join(root, "src")); + // Contains deduped join "alpha beta" but not full repeated phrase "alpha beta alpha". + await fs.writeFile(path.join(root, "src", "repeat.ts"), "export const value = 'alpha beta gamma';\n"); + const response = await searchCodegraph({ + root, + query: "alpha beta alpha", + mode: "text", + limit: 10, + }); + const hit = response.results.find((result) => result.file === "src/repeat.ts"); + expect(hit).toBeTruthy(); + expect(hit?.rankReasons.some((reason) => /exact phrase/i.test(reason))).toBe(false); + }); + it("keeps implementation results ahead of documentation phrases in hybrid mode", async () => { const root = await mkRepo(); diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 126d6953..314a1572 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -118,6 +118,32 @@ describe("navigation package cache invalidation", () => { await fsp.rm(root, { recursive: true, force: true }); } }); + + it("rebuilds cache-off indexes after a same-metadata source mutation", async () => { + const root = await mkTmpDir("codegraph-cache-off-signature-"); + const sourceFile = path.join(root, "source.ts"); + const firstSource = "export const first = 1;\n"; + const secondSource = "export const nextt = 2;\n"; + try { + await fsp.writeFile(sourceFile, firstSource, "utf8"); + const firstIndex = await buildProjectIndex(root, { cache: "off" }); + expect(moduleForPath(firstIndex, sourceFile)?.locals.some((local) => local.localName === "first")).toBe(true); + + const originalStat = await fsp.stat(sourceFile); + await fsp.writeFile(sourceFile, secondSource, "utf8"); + await fsp.utimes(sourceFile, originalStat.atime, originalStat.mtime); + const changedStat = await fsp.stat(sourceFile); + expect(changedStat.size).toBe(originalStat.size); + expect(Math.abs(changedStat.mtimeMs - originalStat.mtimeMs)).toBeLessThan(3); + + const rebuiltIndex = await buildProjectIndex(root, { cache: "off" }); + const rebuiltModule = moduleForPath(rebuiltIndex, sourceFile); + expect(rebuiltModule?.locals.some((local) => local.localName === "nextt")).toBe(true); + expect(rebuiltModule?.locals.some((local) => local.localName === "first")).toBe(false); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); }); function projectSnapshotPathFor(root: string): string { diff --git a/tests/project-file-discovery.test.ts b/tests/project-file-discovery.test.ts index 21cf0404..c0b7ae7e 100644 --- a/tests/project-file-discovery.test.ts +++ b/tests/project-file-discovery.test.ts @@ -690,6 +690,27 @@ describe("project file discovery", () => { expect(overriddenSet.has(normalize(kept))).toBe(false); }); + it("traverses a safe symlink rooted in an explicitly included ignored directory", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "codegraph-project-link-include-override-")); + const packageDir = path.join(tempDir, "packages", "core"); + const linkedPackage = path.join(tempDir, "node_modules"); + const linkedFile = path.join(linkedPackage, "src", "index.ts"); + await createFile(path.join(packageDir, "src", "index.ts"), "export const core = 1;\n"); + try { + await fs.symlink(packageDir, linkedPackage, "junction"); + } catch (error) { + if (isSymlinkUnavailable(error)) return; + throw error; + } + + const discovered = await listProjectFiles(tempDir, ["**/*.ts"], { + includeGlobs: ["node_modules/**"], + useGitignore: false, + }); + + expect(discovered.map(normalize)).toEqual([normalize(linkedFile)]); + }); + it("supports disabling .gitignore filtering and applying additive include/ignore globs", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "codegraph-project-discovery-")); const appFile = path.join(tempDir, "src", "app.ts"); diff --git a/tests/query-index.test.ts b/tests/query-index.test.ts index af43e697..f5160139 100644 --- a/tests/query-index.test.ts +++ b/tests/query-index.test.ts @@ -132,6 +132,17 @@ describe("persistent query index", () => { expect(authEvidence?.line).toBe(1); }); + it("reports both candidate files and candidate chunks for sidecar searches", async () => { + const root = await createRepo(); + const session = createSession(root); + + await search(session, root, "validateUser"); + const diagnostics = (await session.loadProject()).buildReport?.queryIndex; + + expect(diagnostics?.fileCandidates).toBe(2); + expect(diagnostics?.chunkCandidates).toBeGreaterThan(0); + }); + it("keeps compressed indexed text below the amplification target", async () => { const root = await createRepo(); const source = Array.from( From 72ee6b9183aec84d7eac611a3e985741f3872e1e Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 13:41:48 -0400 Subject: [PATCH 15/28] perf(cache): batch writes and hydrate bloom sidecar --- src/chunking/chunkFile.ts | 54 +++++++++- src/duplicates/unitCache.ts | 65 ++++++++++++ src/duplicates/units.ts | 49 +++++++++- src/indexer/build-cache.ts | 5 + src/indexer/build-cache/module-cache.ts | 60 +++++++++--- src/indexer/build-cache/project-snapshot.ts | 103 ++++++++++++++++++++ src/indexer/build-index.ts | 49 +++++++--- tests/cache-invalidation.test.ts | 3 + tests/disk-cache-sqlite.test.ts | 46 ++++++++- 9 files changed, 397 insertions(+), 37 deletions(-) diff --git a/src/chunking/chunkFile.ts b/src/chunking/chunkFile.ts index f845e90a..704cf520 100644 --- a/src/chunking/chunkFile.ts +++ b/src/chunking/chunkFile.ts @@ -39,7 +39,12 @@ export interface ChunkFileOptions { * @param opts Chunking options * @returns Array of semantic chunks */ -export function chunkFile(opts: ChunkFileOptions): Chunk[] { +export type ChunkFileWithSymbolsResult = { + chunks: Chunk[]; + symbolChunks: Chunk[]; +}; + +export function chunkFileWithSymbols(opts: ChunkFileOptions): ChunkFileWithSymbolsResult { const { language, source, filePath, minTokens = 150, maxTokens = 400, tokenizer = countWhitespaceTokens } = opts; const matches = getChunkMatches(language, source, filePath); const newlineOffsets = collectNewlineOffsets(source); @@ -79,7 +84,43 @@ export function chunkFile(opts: ChunkFileOptions): Chunk[] { } preliminaryChunks.sort((left, right) => left.sourceStart - right.sourceStart || right.sourceEnd - left.sourceEnd); - const mergedChunks = mergeSmallChunks(preliminaryChunks, minTokens, maxTokens, tokenizer); + + if (minTokens <= 1) { + const completeChunks = fillGapsWithMiscChunks( + preliminaryChunks, + source, + language.id, + filePath, + tokenizer, + 1, + maxTokens, + newlineOffsets, + ); + const chunks = withStableChunkIds( + completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), + language.id, + filePath, + ); + return { chunks, symbolChunks: chunks }; + } + + const completeSymbolChunks = fillGapsWithMiscChunks( + preliminaryChunks, + source, + language.id, + filePath, + tokenizer, + 1, + maxTokens, + newlineOffsets, + ); + const symbolChunks = withStableChunkIds( + completeSymbolChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), + language.id, + filePath, + ); + + const mergedChunks = mergeSmallChunks([...preliminaryChunks], minTokens, maxTokens, tokenizer); const completeChunks = fillGapsWithMiscChunks( mergedChunks, source, @@ -90,12 +131,17 @@ export function chunkFile(opts: ChunkFileOptions): Chunk[] { maxTokens, newlineOffsets, ); - - return withStableChunkIds( + const chunks = withStableChunkIds( completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), language.id, filePath, ); + + return { chunks, symbolChunks }; +} + +export function chunkFile(opts: ChunkFileOptions): Chunk[] { + return chunkFileWithSymbols(opts).chunks; } function appendBlockChunks( diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 40f1e435..d768d04c 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -370,6 +370,71 @@ export function tryLoadDuplicateUnitsFromCache( } +export type PendingDuplicateUnitCacheWrite = { + file: string; + variant: string; + units: DuplicateInternalUnit[]; +}; + +export function writeDuplicateUnitsBatchToCache( + index: ProjectIndex, + writes: readonly PendingDuplicateUnitCacheWrite[], +): void { + if (!writes.length) return; + const root = index.projectRoot ?? ""; + if (index.cacheMode === "memory") { + for (const write of writes) { + const sig = duplicateUnitCacheSignature(index, write.file); + if (!sig) continue; + writeDuplicateUnitMemoryCache(duplicateUnitCacheKey(write.file, write.variant), { + sig, + units: write.units, + }); + } + return; + } + if (index.cacheMode !== "disk") return; + try { + const entry = duplicateUnitDiskCache(index); + const preparedWrites: Array<{ + file: string; + variant: string; + sig: string; + payload: Buffer; + }> = []; + for (const write of writes) { + const sig = duplicateUnitCacheSignature(index, write.file); + if (!sig) continue; + const payload = brotliCompressSync( + JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(write.units), true)), + { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }, + ); + preparedWrites.push({ + file: cacheRelativePath(root, write.file), + variant: write.variant, + sig, + payload, + }); + } + if (!preparedWrites.length || !entry?.db || !entry.statements) return; + const now = Date.now(); + entry.db.transaction(() => { + for (const write of preparedWrites) { + entry.statements?.write.run( + write.file, + write.variant, + write.sig, + DUPLICATE_UNIT_CACHE_VERSION, + write.payload, + now, + ); + } + })(); + } catch { + // best-effort cache + } +} + export function writeDuplicateUnitsToCache( index: ProjectIndex, file: string, diff --git a/src/duplicates/units.ts b/src/duplicates/units.ts index 8c10a8d6..71cac385 100644 --- a/src/duplicates/units.ts +++ b/src/duplicates/units.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import fsp from "node:fs/promises"; import path from "node:path"; import { LANG_CONFIGS } from "../bootstrap/treeSitterLanguages.js"; -import { chunkFile, type Chunk } from "../chunking/chunkFile.js"; +import { chunkFile, chunkFileWithSymbols, type Chunk } from "../chunking/chunkFile.js"; import { chunkTextFile } from "../chunking/chunkTextFile.js"; import { countDuplicateTokens, @@ -25,7 +25,13 @@ import { maskJsLikeCommentsStringsAndRegex } from "../util/comments.js"; import { collectLineStartOffsets } from "../util/lines.js"; import { assertFilePathWithinRoot, fileIdentityKey, normalizePath, toProjectDisplayPath } from "../util/paths.js"; import { logWithLevel } from "../logging.js"; -import { duplicateUnitCacheVariant, tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "./unitCache.js"; +import { + duplicateUnitCacheVariant, + tryLoadDuplicateUnitsFromCache, + writeDuplicateUnitsBatchToCache, + writeDuplicateUnitsToCache, + type PendingDuplicateUnitCacheWrite, +} from "./unitCache.js"; import type { CollectedDuplicateUnits, DuplicateAstContext, @@ -290,6 +296,29 @@ export function makeDuplicateChunks( return chunkTextFile({ source, filePath, languageId, minTokens, maxTokens, tokenizer: countDuplicateTokens }); } +export function makeDuplicateChunksWithSymbols( + filePath: string, + languageId: string, + textOnly: boolean, + source: string, + minTokens: number, + maxTokens: number, +): { chunks: Chunk[]; symbolChunks: Chunk[] } { + const langConfig = LANG_CONFIGS[chunkLanguageAliases[languageId] ?? languageId]; + if (langConfig && !textOnly) { + return chunkFileWithSymbols({ + language: langConfig, + source, + filePath, + minTokens, + maxTokens, + tokenizer: countDuplicateTokens, + }); + } + const chunks = chunkTextFile({ source, filePath, languageId, minTokens, maxTokens, tokenizer: countDuplicateTokens }); + return { chunks, symbolChunks: [] }; +} + export function makeSymbolSourceChunks( filePath: string, languageId: string, @@ -507,6 +536,7 @@ export async function collectDuplicateUnits( let belowThresholdUnits = 0; const belowThresholdUnitsByFile = new Map(); + const pendingWrites: PendingDuplicateUnitCacheWrite[] = []; for (const file of normalizedFiles) { const cachedUnits = tryLoadDuplicateUnitsFromCache(index, file, variant, options.projectRoot); const fileUnits = @@ -522,7 +552,7 @@ export async function collectDuplicateUnits( astContextCache, )); if (!cachedUnits) { - writeDuplicateUnitsToCache(index, file, variant, fileUnits, options.projectRoot); + pendingWrites.push({ file, variant, units: fileUnits }); } for (const unit of fileUnits) { if (!shouldKeepUnit(unit, options.includeSmall, options.minTokens)) { @@ -533,6 +563,9 @@ export async function collectDuplicateUnits( units.push(unit); } } + if (pendingWrites.length) { + writeDuplicateUnitsBatchToCache(index, pendingWrites); + } units.sort((left, right) => { const fileCompare = left.absoluteFile.localeCompare(right.absoluteFile); @@ -568,8 +601,14 @@ export async function buildDuplicateUnitsForFile( } const astContext = language.textOnly ? undefined : await getDuplicateAstContext(index, file, source, astContextCache); - const chunks = makeDuplicateChunks(file, language.id, language.textOnly, source, minTokens, maxTokens); - const symbolChunks = makeSymbolSourceChunks(file, language.id, language.textOnly, source, maxTokens); + const { chunks, symbolChunks } = makeDuplicateChunksWithSymbols( + file, + language.id, + language.textOnly, + source, + minTokens, + maxTokens, + ); const symbolUnits = (moduleIndex?.locals ?? []) .map((symbol) => { const chunk = findChunkForSymbol(symbol, symbolChunks); diff --git a/src/indexer/build-cache.ts b/src/indexer/build-cache.ts index 9eb08dd2..412bb87d 100644 --- a/src/indexer/build-cache.ts +++ b/src/indexer/build-cache.ts @@ -21,14 +21,19 @@ export { fileSignature, pruneDiskModuleCache, tryLoadFromCache, + writeModulesToCache, writeToCache, type FileSignature, + type PendingModuleCacheWrite, } from "./build-cache/module-cache.js"; export { + BLOOM_FILTER_SNAPSHOT_FILENAME, + BLOOM_FILTER_SNAPSHOT_VERSION, createProjectSnapshotIdentity, projectSnapshotFilesSignature, tryLoadDetailedSymbolGraphSnapshot, tryLoadPersistedBloomFilters, + tryLoadProjectSnapshotModules, tryLoadProjectIndexSnapshot, writeDetailedSymbolGraphSnapshot, writeProjectIndexSnapshot, diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 1131c566..f9d84654 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -244,7 +244,7 @@ function ensureModuleCacheSchema(db: SqliteDatabase, projectRoot: string): void db.exec("CREATE INDEX IF NOT EXISTS idx_module_cache_sig ON module_cache(sig);"); } -function getDiskModuleCache(projectRoot: string, opts?: BuildOptions): DiskModuleCache { +export function getDiskModuleCache(projectRoot: string, opts?: BuildOptions): DiskModuleCache { const dbPath = diskCacheDatabasePath(projectRoot, opts); const existing = diskModuleCaches.get(dbPath); if (existing) return existing; @@ -532,28 +532,48 @@ export function tryLoadFromCache( return null; } -export function writeToCache( +export type PendingModuleCacheWrite = { + file: string; + sig: string; + mod: ModuleIndex; +}; + +export function writeModulesToCache( projectRoot: string, - file: string, - sig: string, - mod: ModuleIndex, + writes: readonly PendingModuleCacheWrite[], opts?: BuildOptions, ): void { + if (!writes.length) return; const mode = opts?.cache ?? "off"; if (mode === "memory") { - lruMapSet( - memoryCache, - memoryCacheKey(projectRoot, file), - { version: PARSED_CACHE_VERSION, sig, mod }, - MAX_MEMORY_CACHE_ENTRIES, - ); + for (const write of writes) { + lruMapSet( + memoryCache, + memoryCacheKey(projectRoot, write.file), + { version: PARSED_CACHE_VERSION, sig: write.sig, mod: write.mod }, + MAX_MEMORY_CACHE_ENTRIES, + ); + } } else if (mode === "disk") { try { const cache = getDiskModuleCache(projectRoot, opts); - const payload = brotliCompressSync(JSON.stringify(transformModulePaths(projectRoot, mod, true)), { - params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, - }); - cache.write.run(cacheRelativePath(projectRoot, file), sig, PARSED_CACHE_VERSION, payload, Date.now()); + const now = Date.now(); + const preparedWrites: Array<{ file: string; sig: string; payload: Buffer }> = []; + for (const write of writes) { + const payload = brotliCompressSync(JSON.stringify(transformModulePaths(projectRoot, write.mod, true)), { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }); + preparedWrites.push({ + file: cacheRelativePath(projectRoot, write.file), + sig: write.sig, + payload, + }); + } + cache.db.transaction(() => { + for (const item of preparedWrites) { + cache.write.run(item.file, item.sig, PARSED_CACHE_VERSION, item.payload, now); + } + })(); } catch (error) { if (isNodeSqliteUnavailableError(error)) { reportMissingNodeSqlite(opts?.logLevel, error); @@ -563,3 +583,13 @@ export function writeToCache( } } } + +export function writeToCache( + projectRoot: string, + file: string, + sig: string, + mod: ModuleIndex, + opts?: BuildOptions, +): void { + writeModulesToCache(projectRoot, [{ file, sig, mod }], opts); +} diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 0094ce54..10cda90f 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -36,6 +36,16 @@ import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); const PROJECT_SNAPSHOT_VERSION = 8; +export const BLOOM_FILTER_SNAPSHOT_VERSION = 1; +export const BLOOM_FILTER_SNAPSHOT_FILENAME = "bloom-filters.json"; + +export type BloomFilterSnapshotPayload = { + version: number; + projectRoot: string; + implementationFingerprint: string; + projectSnapshotIdentity: string; + bloomFilters: Record; +}; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; @@ -461,11 +471,54 @@ export async function tryLoadProjectIndexSnapshot( * `graph.edges` / `modules`. Returns `null` when disk caching is off, `useBloomFilters` is * disabled, or no valid snapshot with bloom data exists. */ +export async function tryLoadProjectSnapshotModules( + projectRoot: string, + opts: BuildOptions | undefined, +): Promise | null> { + if ((opts?.cache ?? "off") !== "disk") return null; + try { + const rawPayload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; + const migratedPayload = migrateProjectSnapshotPayload(rawPayload, projectRoot); + const payload = + migratedPayload && typeof migratedPayload === "object" && !Array.isArray(migratedPayload) + ? transformSnapshotPaths(migratedPayload as ProjectIndexSnapshotPayload, projectRoot, false) + : migratedPayload; + const nativeRuntimeFingerprint = getNativeRuntimeFingerprint(opts?.native); + const implementationFingerprint = getImplementationFingerprint(); + if ( + !isProjectIndexSnapshotPayload(payload) || + !projectRootMatches(projectRoot, payload.projectRoot) || + payload.nativeMode !== normalizedSnapshotNativeMode(opts?.native) || + payload.nativeRuntimeFingerprint !== nativeRuntimeFingerprint || + payload.implementationFingerprint !== implementationFingerprint + ) { + return null; + } + const modules = new Map(); + for (const mod of payload.modules) { + modules.set(fileIdentityKey(mod.file), mod); + } + return modules; + } catch { + return null; + } +} + export async function tryLoadPersistedBloomFilters( projectRoot: string, opts: BuildOptions | undefined, ): Promise { if ((opts?.cache ?? "off") !== "disk" || (opts?.useBloomFilters ?? true) === false) return null; + try { + const sidecarPath = bloomFilterSnapshotPath(projectRoot, opts); + const sidecarParsed = (await readParsedSnapshot(sidecarPath)).payload; + const sidecarBloom = persistedBloomFiltersFromSidecar(sidecarParsed, projectRoot); + if (sidecarBloom) { + return deserializeBloomFilterCache(sidecarBloom, projectRoot); + } + } catch { + // Fall back to legacy project snapshot payload if sidecar is unavailable or corrupt + } try { const payload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; const bloomFilters = persistedBloomFiltersFromSnapshot(payload, projectRoot); @@ -476,6 +529,24 @@ export async function tryLoadPersistedBloomFilters( } } +function persistedBloomFiltersFromSidecar( + value: unknown, + projectRoot: string, +): Record | null { + if (!value || typeof value !== "object") return null; + const payload = value as Partial; + if ( + payload.version !== BLOOM_FILTER_SNAPSHOT_VERSION || + typeof payload.projectRoot !== "string" || + !projectRootMatches(projectRoot, payload.projectRoot) || + payload.implementationFingerprint !== getImplementationFingerprint() + ) { + return null; + } + if (!isSerializedBloomFilterRecord(payload.bloomFilters)) return null; + return payload.bloomFilters; +} + /** Light validation for bloom hydration: snapshot version, root identity, and bloom section only. */ function persistedBloomFiltersFromSnapshot( value: unknown, @@ -544,6 +615,29 @@ export async function writeProjectIndexSnapshot( const identity = await snapshotFileIdentity(snapshotPath); setBoundedSnapshotCache(parsedSnapshotCache, snapshotPath, { identity, compressed }); index.projectSnapshotIdentity = projectSnapshotIdentity; + if (serializedBloomFilters) { + try { + const bloomPayload: BloomFilterSnapshotPayload = { + version: BLOOM_FILTER_SNAPSHOT_VERSION, + projectRoot: serializedProjectRoot(projectRoot), + implementationFingerprint: getImplementationFingerprint(), + projectSnapshotIdentity, + bloomFilters: serializedBloomFilters, + }; + const bloomPath = bloomFilterSnapshotPath(projectRoot, opts); + const bloomCompressed = brotliCompressSync(JSON.stringify(bloomPayload), { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }); + await writeSnapshotAtomically(bloomPath, bloomCompressed); + const bloomIdentity = await snapshotFileIdentity(bloomPath); + setBoundedSnapshotCache(parsedSnapshotCache, bloomPath, { + identity: bloomIdentity, + compressed: bloomCompressed, + }); + } catch { + // Bloom sidecar write failure must not fail indexing + } + } } catch { delete index.projectSnapshotIdentity; // Snapshot writes are an optimization; cache write failures must not fail indexing. @@ -554,6 +648,15 @@ function projectSnapshotPath(projectRoot: string, opts: BuildOptions | undefined return path.join(cacheRoot(projectRoot, opts), "project-index-snapshot.json"); } +function bloomFilterSnapshotPath(projectRoot: string, opts: BuildOptions | undefined): string { + const root = path.resolve(cacheRoot(projectRoot, opts)); + const snapshotPath = path.resolve(root, BLOOM_FILTER_SNAPSHOT_FILENAME); + if (!isFilePathWithinRoot(root, snapshotPath)) { + throw new Error(`Bloom filter snapshot escaped cache root: ${snapshotPath}`); + } + return snapshotPath; +} + export async function tryLoadDetailedSymbolGraphSnapshot( projectRoot: string, opts: BuildOptions | undefined, diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 2b504603..a3e5aca3 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -65,11 +65,14 @@ import { tryLoadFromCache, tryLoadPersistedBloomFilters, tryLoadProjectIndexSnapshot, + tryLoadProjectSnapshotModules, verifyManifestEntries, + writeModulesToCache, writeProjectIndexSnapshot, writeToCache, type FileSignature, type ManifestFileEntry, + type PendingModuleCacheWrite, } from "./build-cache.js"; import { cacheRoot } from "./build-cache/module-cache.js"; import { @@ -120,6 +123,7 @@ type IndexedFileGraphContext = { type IndexedFileModuleResult = { module: ModuleIndex; + cacheWrite?: PendingModuleCacheWrite | undefined; graphContext: IndexedFileGraphContext; }; @@ -336,15 +340,17 @@ async function buildIndexedModuleForFile(args: { const sigInfo = args.fileSignatures.get(args.file); const cacheable = !prepared.nativeFallbackReason && !lacksParserContext; + let cacheWrite: PendingModuleCacheWrite | undefined; if (sigInfo && cacheable) { const cacheSig = args.cacheEnabled ? await moduleCacheSignatureForFile(args.file, sigInfo, args.opts) : sigInfo.cacheSig; - writeToCache(args.projectRoot, args.file, cacheSig, mod, args.opts); + cacheWrite = { file: args.file, sig: cacheSig, mod }; } return { module: mod, + ...(cacheWrite !== undefined ? { cacheWrite } : {}), graphContext: { source, sup, @@ -784,13 +790,14 @@ async function buildIndexFromFileListShared( if (filter) bloomFilterCache.set(file, filter); } } - return [file, mod, edges] as const; + return [file, mod, edges, undefined] as const; } if (fileReport) fileReport.parsed = (fileReport.parsed ?? 0) + 1; const support = supportForFile(file, opts?.languageExtensions); - if (!support) return [file, createEmptyModuleIndex(file), []] as const; + if (!support) return [file, createEmptyModuleIndex(file), [], undefined] as const; ensureBuildProgressStarted(); let graphContext: IndexedFileGraphContext | undefined; + let cacheWrite: PendingModuleCacheWrite | undefined; if (!mod) { const built = await buildIndexedModuleForFile({ file, @@ -811,6 +818,7 @@ async function buildIndexFromFileListShared( }); mod = built.module; graphContext = built.graphContext; + cacheWrite = built.cacheWrite; } else { collectJsonDependencies(mod.imports, jsonDependencies); } @@ -835,11 +843,11 @@ async function buildIndexFromFileListShared( allFiles: normalizedFiles, ...(sqlFactCache ? { sqlFactCache } : {}), }); - return [file, mod ?? createEmptyModuleIndex(file), edges] as const; + return [file, mod ?? createEmptyModuleIndex(file), edges, cacheWrite] as const; } catch (error) { if (isNativeRequiredUnavailableError(error) || isNodeSqliteUnavailableError(error)) throw error; if (isUnsupportedParserInputError(error) || isNonNativeParserUnavailableError(error)) { - return [file, createEmptyModuleIndex(file), []] as const; + return [file, createEmptyModuleIndex(file), [], undefined] as const; } recordFileFailure(report, file, error); logWithLevel(opts?.logLevel, "warn", `Warning: Failed to process file ${file}:`, error); @@ -871,9 +879,14 @@ async function buildIndexFromFileListShared( if (edge.to.type === "file") graph.nodes.add(edge.to.path); } }; - for (const [file, mod, edges] of fileResults) { + const pendingCacheWrites: PendingModuleCacheWrite[] = []; + for (const [file, mod, edges, cacheWrite] of fileResults) { modules.set(fileIdentityKey(file), mod); appendUniqueGraphEdges(edges); + if (cacheWrite) pendingCacheWrites.push(cacheWrite); + } + if (pendingCacheWrites.length) { + writeModulesToCache(projectRoot, pendingCacheWrites, opts); } const workspaceManifestEdges = await collectWorkspaceManifestDependencyEdges( projectRoot, @@ -1454,7 +1467,7 @@ export async function buildProjectIndexIncremental( opts, gitSigMap, cacheEnabled, - needsContentHash: cacheEnabled || opts?.cacheStrict !== false, + needsContentHash: cacheEnabled || manifestUsed || opts?.cacheStrict === true, concurrency: conc, }); const modules = new Map(); @@ -1499,12 +1512,19 @@ export async function buildProjectIndexIncremental( completeCheckProgress(allFiles.size); return unchangedSnapshot; } + const snapshotModules = cacheEnabled ? await tryLoadProjectSnapshotModules(projectRoot, opts) : null; const persistedBloomFilters = bloomFilterCache ? await tryLoadPersistedBloomFilters(projectRoot, opts) : null; for (const file of allFiles) { if (changedFiles.has(file)) continue; const sigInfo = fileSignatures.get(file)!; - const cacheSig = cacheEnabled ? await moduleCacheSignatureForFile(file, sigInfo, opts) : sigInfo.cacheSig; - const cached = cacheEnabled ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) : null; + let cached: ModuleIndex | null = null; + const snapshotMod = snapshotModules?.get(fileIdentityKey(file)); + if (snapshotMod) { + cached = snapshotMod; + } else if (cacheEnabled) { + const cacheSig = await moduleCacheSignatureForFile(file, sigInfo, opts); + cached = tryLoadFromCache(projectRoot, file, cacheSig, opts, report); + } if (cached) { if (fileReport) fileReport.cached = (fileReport.cached ?? 0) + 1; modules.set(fileIdentityKey(file), cached); @@ -1556,7 +1576,7 @@ export async function buildProjectIndexIncremental( fileSignatures, cacheEnabled, }); - return [file, built.module] as const; + return [file, built.module, built.cacheWrite] as const; } catch (error) { if (isNativeRequiredUnavailableError(error) || isNodeSqliteUnavailableError(error)) throw error; if (isUnsupportedParserInputError(error) || isNonNativeParserUnavailableError(error)) { @@ -1564,7 +1584,7 @@ export async function buildProjectIndexIncremental( } recordFileFailure(report, file, error); logWithLevel(opts?.logLevel, "warn", `Warning: Failed to process file ${file}:`, error); - return [file, createEmptyModuleIndex(file)] as const; + return [file, createEmptyModuleIndex(file), undefined] as const; } finally { if (opts?.onProgress) { opts.onProgress({ @@ -1578,8 +1598,13 @@ export async function buildProjectIndexIncremental( } } }); - for (const [file, mod] of fileResults) { + const pendingIncrementalCacheWrites: PendingModuleCacheWrite[] = []; + for (const [file, mod, cacheWrite] of fileResults) { modules.set(fileIdentityKey(file), mod); + if (cacheWrite) pendingIncrementalCacheWrites.push(cacheWrite); + } + if (pendingIncrementalCacheWrites.length) { + writeModulesToCache(projectRoot, pendingIncrementalCacheWrites, opts); } if (timings) timings.parseMs = Math.round(performance.now() - parseStart); } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 314a1572..744f06d5 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -1820,6 +1820,9 @@ describe("Cache invalidation and strict hashing", () => { await fsp.writeFile(gammaPath, "export const gammaValue = 3;\n", "utf8"); await buildProjectIndex(root, { threads: 2, cache: "disk", useBloomFilters: true }); + const bloomSidecarPath = path.join(root, ".codegraph-cache", "index-v1", "bloom-filters.json"); + expect(await fsp.stat(bloomSidecarPath)).toBeTruthy(); + await fsp.rm(bloomSidecarPath); // Modify only gamma.ts: alpha.ts and beta.ts stay genuine, provable cache hits, but the // snapshot as a whole can no longer be reused wholesale (`changedFiles.size` is nonzero), diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 0f5cbc21..75b2b982 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -8,7 +8,8 @@ import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/inde import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; import { tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "../src/duplicates/unitCache.js"; import { buildInternalUnit, formatDuplicateSqlHandle, formatDuplicateSymbolHandle } from "../src/duplicates/units.js"; -import { SqliteDatabase } from "../src/sqlite-driver.js"; +import * as buildCache from "../src/indexer/build-cache.js"; +import { SqliteDatabase, SqliteStatement } from "../src/sqlite-driver.js"; import { mkTmpDir } from "./helpers/filesystem.js"; function cacheDir(root: string): string { @@ -144,6 +145,49 @@ describe("disk cache uses sqlite backend", () => { ).toHaveLength(1); expect(sql.filter((statement) => statement.startsWith("INSERT INTO module_cache"))).toHaveLength(1); }); + it("rolls back a failed module cache batch without a partial row", async () => { + const root = await mkTmpDir("dg-disk-cache-batch-rollback-"); + const firstPath = path.join(root, "first.ts"); + const secondPath = path.join(root, "second.ts"); + await fsp.writeFile(firstPath, "export const first = 1;\n", "utf8"); + await fsp.writeFile(secondPath, "export const second = 2;\n", "utf8"); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const first = Array.from(index.byFile.values()).find((mod) => mod.file.endsWith("/first.ts")); + const second = Array.from(index.byFile.values()).find((mod) => mod.file.endsWith("/second.ts")); + if (!first || !second) throw new Error("missing seeded modules"); + + const db = new DatabaseSync(moduleCacheDbPath(root)); + db.exec("DELETE FROM module_cache;"); + db.close(); + + const originalRun = SqliteStatement.prototype.run; + let cacheWrites = 0; + const runSpy = vi.spyOn(SqliteStatement.prototype, "run").mockImplementation(function ( + this: SqliteStatement, + ...params + ) { + if (params.length === 5 && params[2] === 4) { + cacheWrites++; + if (cacheWrites === 2) throw new Error("simulated aborted cache batch"); + } + return originalRun.call(this, ...params); + }); + try { + buildCache.writeModulesToCache( + root, + [ + { file: firstPath, sig: "first-signature", mod: first }, + { file: secondPath, sig: "second-signature", mod: second }, + ], + { cache: "disk" }, + ); + } finally { + runSpy.mockRestore(); + } + + expect(cacheWrites).toBe(2); + expect(readRowCount(moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache")).toBe(0); + }); it("prunes module cache rows for files outside the successful manifest", async () => { const root = await mkTmpDir("dg-disk-cache-prune-"); From 32eef3f986452288d54233404efb03c0d31ba1b9 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 22:52:06 -0400 Subject: [PATCH 16/28] fix(cache): validate hydrated snapshot artifacts --- src/indexer/build-cache/project-snapshot.ts | 128 +++++++++++++++++--- src/indexer/build-index.ts | 8 +- tests/cache-invalidation.test.ts | 74 +++++++++++ 3 files changed, 191 insertions(+), 19 deletions(-) diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 10cda90f..d2f55757 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -31,12 +31,18 @@ import { type SymbolVisibility, } from "../../graphs/symbol-graph.js"; import { getImplementationFingerprint, normalizeGraphOptions } from "./options.js"; -import { cacheAbsolutePath, cacheRelativePath, cacheRoot, transformPersistedExportFromModule } from "./module-cache.js"; +import { + cacheAbsolutePath, + cacheRelativePath, + cacheRoot, + transformPersistedExportFromModule, + type FileSignature, +} from "./module-cache.js"; import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); const PROJECT_SNAPSHOT_VERSION = 8; -export const BLOOM_FILTER_SNAPSHOT_VERSION = 1; +export const BLOOM_FILTER_SNAPSHOT_VERSION = 2; export const BLOOM_FILTER_SNAPSHOT_FILENAME = "bloom-filters.json"; export type BloomFilterSnapshotPayload = { @@ -44,6 +50,7 @@ export type BloomFilterSnapshotPayload = { projectRoot: string; implementationFingerprint: string; projectSnapshotIdentity: string; + fileSignatures: Record; bloomFilters: Record; }; const BLOOM_FILTER_MIN_SIZE = 1_000; @@ -94,6 +101,15 @@ type SerializedBloomFilter = { bitsBase64: string; }; +type SnapshotFileSignature = { + sig: string; + gitSig?: string; +}; + +export type PersistedBloomFilters = { + get: (file: string, signature: Pick) => BloomFilter | undefined; +}; + type SnapshotAnalysisReport = { backend?: BackendReport; graph?: GraphReport; @@ -119,6 +135,7 @@ type ProjectIndexSnapshotPayload = { implementationFingerprint: string; projectFiles?: ProjectFileInfo[]; bloomFilters?: Record; + fileSignatures: Record; analysis?: AnalysisSummary; analysisReport?: SnapshotAnalysisReport; }; @@ -217,6 +234,11 @@ function transformSnapshotPaths( } copy.bloomFilters = bloomFilters; } + const fileSignatures: Record = {}; + for (const [file, signature] of Object.entries(copy.fileSignatures)) { + fileSignatures[transformPath(root, file, toRelative)] = signature; + } + copy.fileSignatures = fileSignatures; return copy; } function migrateProjectSnapshotPayload(value: unknown, currentRoot: string): unknown { @@ -463,10 +485,9 @@ export async function tryLoadProjectIndexSnapshot( } /** - * Load only the bloom-filter section of the last-written project snapshot, without requiring - * the whole snapshot to match this build's files signature or native runtime fingerprint. - * Bloom filters are a pure function of a file's text, so a filter persisted for a given file - * is safe to reuse after its snapshot's root and implementation fingerprints are validated. + * Load only the bloom-filter section of the last-written project snapshot. Hydration checks + * each requested file against its persisted signature before returning a filter, so a stale + * snapshot can still accelerate unchanged files without suppressing newly added references. * Only the bloom section is then read, so a corrupt payload is rejected without walking * `graph.edges` / `modules`. Returns `null` when disk caching is off, `useBloomFilters` is * disabled, or no valid snapshot with bloom data exists. @@ -474,6 +495,7 @@ export async function tryLoadProjectIndexSnapshot( export async function tryLoadProjectSnapshotModules( projectRoot: string, opts: BuildOptions | undefined, + fileSignatures: ReadonlyMap>, ): Promise | null> { if ((opts?.cache ?? "off") !== "disk") return null; try { @@ -496,6 +518,9 @@ export async function tryLoadProjectSnapshotModules( } const modules = new Map(); for (const mod of payload.modules) { + const signature = fileSignatures.get(fileIdentityKey(mod.file)); + const snapshotSignature = payload.fileSignatures[fileIdentityKey(mod.file)]; + if (!signature || !snapshotSignature || !snapshotSignatureMatches(snapshotSignature, signature)) continue; modules.set(fileIdentityKey(mod.file), mod); } return modules; @@ -507,14 +532,14 @@ export async function tryLoadProjectSnapshotModules( export async function tryLoadPersistedBloomFilters( projectRoot: string, opts: BuildOptions | undefined, -): Promise { +): Promise { if ((opts?.cache ?? "off") !== "disk" || (opts?.useBloomFilters ?? true) === false) return null; try { const sidecarPath = bloomFilterSnapshotPath(projectRoot, opts); const sidecarParsed = (await readParsedSnapshot(sidecarPath)).payload; const sidecarBloom = persistedBloomFiltersFromSidecar(sidecarParsed, projectRoot); if (sidecarBloom) { - return deserializeBloomFilterCache(sidecarBloom, projectRoot); + return createPersistedBloomFilters(sidecarBloom.bloomFilters, sidecarBloom.fileSignatures, projectRoot); } } catch { // Fall back to legacy project snapshot payload if sidecar is unavailable or corrupt @@ -523,7 +548,7 @@ export async function tryLoadPersistedBloomFilters( const payload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; const bloomFilters = persistedBloomFiltersFromSnapshot(payload, projectRoot); if (!bloomFilters) return null; - return deserializeBloomFilterCache(bloomFilters, projectRoot); + return createPersistedBloomFilters(bloomFilters.bloomFilters, bloomFilters.fileSignatures, projectRoot); } catch { return null; } @@ -532,38 +557,79 @@ export async function tryLoadPersistedBloomFilters( function persistedBloomFiltersFromSidecar( value: unknown, projectRoot: string, -): Record | null { +): Pick | null { if (!value || typeof value !== "object") return null; const payload = value as Partial; if ( payload.version !== BLOOM_FILTER_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || !projectRootMatches(projectRoot, payload.projectRoot) || - payload.implementationFingerprint !== getImplementationFingerprint() + payload.implementationFingerprint !== getImplementationFingerprint() || + typeof payload.projectSnapshotIdentity !== "string" || + !/^[a-f0-9]{64}$/.test(payload.projectSnapshotIdentity) ) { return null; } - if (!isSerializedBloomFilterRecord(payload.bloomFilters)) return null; - return payload.bloomFilters; + const bloomFilters = payload.bloomFilters; + const fileSignatures = payload.fileSignatures; + if (!isSerializedBloomFilterRecord(bloomFilters) || !isSnapshotFileSignatureRecord(fileSignatures)) { + return null; + } + return { bloomFilters, fileSignatures }; } /** Light validation for bloom hydration: snapshot version, root identity, and bloom section only. */ function persistedBloomFiltersFromSnapshot( value: unknown, projectRoot: string, -): Record | null { +): Pick | null { const migrated = migrateProjectSnapshotPayload(value, projectRoot); if (!migrated || typeof migrated !== "object") return null; const payload = migrated as Partial; if ( payload.version !== PROJECT_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || + !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() ) { return null; } - if (!isSerializedBloomFilterRecord(payload.bloomFilters)) return null; - return payload.bloomFilters; + const bloomFilters = payload.bloomFilters; + const fileSignatures = payload.fileSignatures; + if (!isSerializedBloomFilterRecord(bloomFilters) || !isSnapshotFileSignatureRecord(fileSignatures)) { + return null; + } + return { bloomFilters, fileSignatures }; +} + +function createPersistedBloomFilters( + bloomFilters: Record, + fileSignatures: Record, + projectRoot: string, +): PersistedBloomFilters { + const filters = deserializeBloomFilterCache(bloomFilters, projectRoot); + const signatures = new Map( + Object.entries(fileSignatures).map(([file, signature]) => [ + fileIdentityKey(cacheAbsolutePath(projectRoot, file)), + signature, + ]), + ); + return { + get: (file, signature) => { + const persistedSignature = signatures.get(fileIdentityKey(file)); + if (!persistedSignature || !snapshotSignatureMatches(persistedSignature, signature)) return undefined; + return filters.get(file); + }, + }; +} + +function snapshotSignatureMatches( + snapshotSignature: SnapshotFileSignature, + currentSignature: Pick, +): boolean { + const matchingGitSignature = + !!snapshotSignature.gitSig && !!currentSignature.gitSig && snapshotSignature.gitSig === currentSignature.gitSig; + return matchingGitSignature || snapshotSignature.sig === currentSignature.sig; } export async function writeProjectIndexSnapshot( @@ -573,6 +639,7 @@ export async function writeProjectIndexSnapshot( filesSignature: string, ): Promise { const projectSnapshotIdentity = createProjectSnapshotIdentity(filesSignature, opts); + const fileSignatures = serializeSnapshotFileSignatures(index.manifestEntries, projectRoot); const serializedBloomFilters = index.bloomFilters ? serializeBloomFilterCache( index.bloomFilters, @@ -594,6 +661,7 @@ export async function writeProjectIndexSnapshot( edges: index.graph.edges, }, modules: [...index.byFile.values()], + fileSignatures, ...(index.languageExtensions ? { languageExtensions: index.languageExtensions } : {}), ...(normalizedSnapshotNativeMode(index.nativeMode) ? { nativeMode: normalizedSnapshotNativeMode(index.nativeMode) } @@ -622,6 +690,7 @@ export async function writeProjectIndexSnapshot( projectRoot: serializedProjectRoot(projectRoot), implementationFingerprint: getImplementationFingerprint(), projectSnapshotIdentity, + fileSignatures, bloomFilters: serializedBloomFilters, }; const bloomPath = bloomFilterSnapshotPath(projectRoot, opts); @@ -1057,6 +1126,7 @@ function isProjectIndexSnapshotPayload(value: unknown): value is ProjectIndexSna payload.graph.edges.every(isGraphEdge) && Array.isArray(payload.modules) && payload.modules.every(isModuleIndex) && + isSnapshotFileSignatureRecord(payload.fileSignatures) && (payload.nativeMode === undefined || isSnapshotNativeMode(payload.nativeMode)) && (payload.bloomFilters === undefined || isSerializedBloomFilterRecord(payload.bloomFilters)) && (payload.analysis === undefined || isAnalysisSummary(payload.analysis)) && @@ -1177,6 +1247,32 @@ function serializeBloomFilterCache( } return Object.keys(serialized).length ? serialized : undefined; } + +function serializeSnapshotFileSignatures( + entries: ProjectIndex["manifestEntries"], + projectRoot: string, +): Record { + const serialized: Record = {}; + for (const [file, entry] of entries ?? []) { + serialized[cacheRelativePath(projectRoot, file)] = { + sig: entry.sig, + ...(entry.gitSig ? { gitSig: entry.gitSig } : {}), + }; + } + return serialized; +} + +function isSnapshotFileSignatureRecord(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + return Object.values(value).every(isSnapshotFileSignature); +} + +function isSnapshotFileSignature(value: unknown): value is SnapshotFileSignature { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const signature = value as Partial; + return typeof signature.sig === "string" && (signature.gitSig === undefined || typeof signature.gitSig === "string"); +} + function deserializeBloomFilterCache( serialized: Record, projectRoot: string, diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index a3e5aca3..f8d1df21 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -782,7 +782,7 @@ async function buildIndexFromFileListShared( ...(sqlFactCache ? { sqlFactCache } : {}), }); if (bloomFilterCache) { - const persistedFilter = persistedBloomFilters?.get(file); + const persistedFilter = persistedBloomFilters?.get(file, sigInfo); if (persistedFilter) { bloomFilterCache.set(file, persistedFilter); } else { @@ -1512,7 +1512,9 @@ export async function buildProjectIndexIncremental( completeCheckProgress(allFiles.size); return unchangedSnapshot; } - const snapshotModules = cacheEnabled ? await tryLoadProjectSnapshotModules(projectRoot, opts) : null; + const snapshotModules = cacheEnabled + ? await tryLoadProjectSnapshotModules(projectRoot, opts, fileSignatures) + : null; const persistedBloomFilters = bloomFilterCache ? await tryLoadPersistedBloomFilters(projectRoot, opts) : null; for (const file of allFiles) { if (changedFiles.has(file)) continue; @@ -1530,7 +1532,7 @@ export async function buildProjectIndexIncremental( modules.set(fileIdentityKey(file), cached); collectJsonDependencies(cached.imports, jsonDependencies); if (bloomFilterCache) { - const persistedFilter = persistedBloomFilters?.get(file); + const persistedFilter = persistedBloomFilters?.get(file, sigInfo); if (persistedFilter) { bloomFilterCache.set(file, persistedFilter); } else { diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 744f06d5..7adf678a 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -9,6 +9,7 @@ import { buildProjectIndex, buildProjectIndexFromFiles, buildProjectIndexIncremental, + findReferences, resolveExport, type BuildReport, } from "../src/index.js"; @@ -1852,6 +1853,79 @@ describe("Cache invalidation and strict hashing", () => { bloomSpy.mockRestore(); }); + it("rejects stale snapshot modules after the manifest has advanced", async () => { + const root = await mkTmpDir("dg-stale-snapshot-modules-"); + const entryPath = path.join(root, "entry.ts"); + await fsp.writeFile(entryPath, "export const staleSnapshotValue = 1;\n", "utf8"); + await buildProjectIndex(root, { threads: 1, cache: "disk", useBloomFilters: true }); + + const snapshotPath = projectSnapshotPathFor(root); + const staleSnapshot = await fsp.readFile(snapshotPath); + await fsp.writeFile(entryPath, "export const currentSnapshotValue = 2;\n", "utf8"); + await buildProjectIndexIncremental(root, { threads: 1, cache: "disk", useBloomFilters: true }); + await fsp.writeFile(snapshotPath, staleSnapshot); + + const recovered = await buildProjectIndexIncremental(root, { + threads: 1, + cache: "disk", + useBloomFilters: true, + }); + const entry = recovered.byFile.get(fileIdentityKey(normalize(entryPath))); + + expect(entry?.locals.some((local) => local.localName === "currentSnapshotValue")).toBe(true); + expect(entry?.locals.some((local) => local.localName === "staleSnapshotValue")).toBe(false); + }); + + it("rejects stale bloom sidecars and recovers semantic references", async () => { + const root = await mkTmpDir("dg-stale-bloom-sidecar-"); + const definitionPath = path.join(root, "definition.ts"); + const consumerPath = path.join(root, "consumer.ts"); + const triggerPath = path.join(root, "trigger.ts"); + await fsp.writeFile(definitionPath, "export class Worker { staleBloomMethod() {} }\n", "utf8"); + await fsp.writeFile( + consumerPath, + 'import { Worker } from "./definition";\nexport const consumer = new Worker().staleBloomMethod();\n', + "utf8", + ); + await fsp.writeFile(triggerPath, "export const trigger = 1;\n", "utf8"); + await buildProjectIndex(root, { threads: 1, cache: "disk", useBloomFilters: true }); + + const snapshotPath = projectSnapshotPathFor(root); + const sidecarPath = path.join(root, ".codegraph-cache", "index-v1", "bloom-filters.json"); + const staleSnapshot = await fsp.readFile(snapshotPath); + const staleSidecar = await fsp.readFile(sidecarPath); + await fsp.writeFile(definitionPath, "export class Worker { currentBloomMethod() {} }\n", "utf8"); + await fsp.writeFile( + consumerPath, + 'import { Worker } from "./definition";\nexport const consumer = new Worker().currentBloomMethod();\n', + "utf8", + ); + await buildProjectIndexIncremental(root, { threads: 1, cache: "disk", useBloomFilters: true }); + await fsp.writeFile(snapshotPath, staleSnapshot); + await fsp.writeFile(sidecarPath, staleSidecar); + await fsp.writeFile(triggerPath, "export const trigger = 2;\n", "utf8"); + + const recovered = await buildProjectIndexIncremental(root, { + threads: 1, + cache: "disk", + useBloomFilters: true, + }); + const definition = recovered.byFile + .get(fileIdentityKey(normalize(definitionPath))) + ?.locals.find((local) => local.localName === "currentBloomMethod"); + if (!definition) throw new Error("Expected current bloom definition"); + + const references = await findReferences(recovered, { def: definition }); + + expect(recovered.bloomFilters?.get(normalize(consumerPath))?.mightContain("currentBloomMethod")).toBe(true); + expect(references.status).toBe("ok"); + if (references.status === "ok") { + expect(references.references.some((reference) => normalize(reference.file) === normalize(consumerPath))).toBe( + true, + ); + } + }); + it("does not hydrate persisted bloom filters when bloom filters are disabled", async () => { const root = await mkTmpDir("dg-snapshot-bloom-disabled-"); const entryPath = path.join(root, "entry.ts"); From 042b0598d9c5325d0cc17dec54fbebbd8433c1cb Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 09:35:42 -0400 Subject: [PATCH 17/28] fix(cache): preserve hydrated bloom validation --- src/indexer/build-cache/project-snapshot.ts | 5 ++++- tests/disk-cache-sqlite.test.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index d2f55757..b297a1e7 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -582,7 +582,10 @@ function persistedBloomFiltersFromSidecar( function persistedBloomFiltersFromSnapshot( value: unknown, projectRoot: string, -): Pick | null { +): { + bloomFilters: Record; + fileSignatures: Record; +} | null { const migrated = migrateProjectSnapshotPayload(value, projectRoot); if (!migrated || typeof migrated !== "object") return null; const payload = migrated as Partial; diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 75b2b982..27f028dc 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -166,7 +166,7 @@ describe("disk cache uses sqlite backend", () => { this: SqliteStatement, ...params ) { - if (params.length === 5 && params[2] === 4) { + if (params.length === 5 && (params[0] === "first.ts" || params[0] === "second.ts")) { cacheWrites++; if (cacheWrites === 2) throw new Error("simulated aborted cache batch"); } From 7ec1fc8c4634e9bcd6b68b724d81c46537c0ec8a Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 09:48:08 -0400 Subject: [PATCH 18/28] fix(cache): keep diagnostics lightweight --- src/cli/doctor.ts | 2 +- src/cli/inspect.ts | 2 +- src/cli/viewer.ts | 2 +- src/indexer/build-cache.ts | 2 +- src/indexer/build-cache/location.ts | 103 ++++++++++++++++++++ src/indexer/build-cache/manifest.ts | 3 +- src/indexer/build-cache/module-cache.ts | 97 +----------------- src/indexer/build-cache/project-snapshot.ts | 2 +- src/indexer/build-index.ts | 2 +- src/indexer/finalize.ts | 2 +- tests/cache-path-confinement.test.ts | 11 ++- 11 files changed, 121 insertions(+), 107 deletions(-) create mode 100644 src/indexer/build-cache/location.ts diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index ead0abe6..5abfaa2b 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import { cacheRoot, resolveCacheAnchor } from "../indexer/build-cache/module-cache.js"; +import { cacheRoot, resolveCacheAnchor } from "../indexer/build-cache/location.js"; import { isNativeTreeSitterAvailable, getNativeBindingOrigin, diff --git a/src/cli/inspect.ts b/src/cli/inspect.ts index 50fe1d6f..a96c616b 100644 --- a/src/cli/inspect.ts +++ b/src/cli/inspect.ts @@ -13,7 +13,7 @@ import { getNativeTreeSitterSupportedLanguageIds, isNativeTreeSitterAvailable, } from "../native/treeSitterNative.js"; -import { cacheRoot } from "../indexer/build-cache/module-cache.js"; +import { cacheRoot } from "../indexer/build-cache/location.js"; import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; import type { Graph } from "../types.js"; import { restrictGraphToIncludeRoots } from "../util/includeRoots.js"; diff --git a/src/cli/viewer.ts b/src/cli/viewer.ts index 0450f8a8..34a3de6a 100644 --- a/src/cli/viewer.ts +++ b/src/cli/viewer.ts @@ -14,7 +14,7 @@ import { isAllowedHostHeader, listenOnHttpServer, } from "../mcp/http.js"; -import { cacheRoot } from "../indexer/build-cache/module-cache.js"; +import { cacheRoot } from "../indexer/build-cache/location.js"; import { parseOptionalBoundedIntegerOption } from "./options.js"; import { assertFilePathWithinRoot, resolveFilePathFromRoot } from "../util/paths.js"; import { errorMessage } from "../util/errors.js"; diff --git a/src/indexer/build-cache.ts b/src/indexer/build-cache.ts index 412bb87d..ddaaf5b7 100644 --- a/src/indexer/build-cache.ts +++ b/src/indexer/build-cache.ts @@ -14,7 +14,6 @@ export { } from "./build-cache/manifest.js"; export { buildBloomFilterForFile, - cacheRoot, cacheSignatureForFile, clearMemoryCache, closeDiskCacheDatabase, @@ -26,6 +25,7 @@ export { type FileSignature, type PendingModuleCacheWrite, } from "./build-cache/module-cache.js"; +export { cacheRoot } from "./build-cache/location.js"; export { BLOOM_FILTER_SNAPSHOT_FILENAME, BLOOM_FILTER_SNAPSHOT_VERSION, diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts new file mode 100644 index 00000000..69a34f3c --- /dev/null +++ b/src/indexer/build-cache/location.ts @@ -0,0 +1,103 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { BuildOptions } from "../types.js"; +import { fileIdentityKey } from "../../util/paths.js"; + +export type CacheAnchorResolution = { + anchor: string; + layer: "explicit" | "environment" | "manifest" | "git" | "project" | "user"; +}; + +function isWritableDirectory(candidate: string): boolean { + try { + return fs.statSync(candidate).isDirectory() && fs.accessSync(candidate, fs.constants.W_OK) === undefined; + } catch { + return false; + } +} + +function isForbiddenAnchor(candidate: string): boolean { + const resolved = path.resolve(candidate); + const home = path.resolve(os.homedir()); + const parsed = path.parse(resolved); + return ( + fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root) + ); +} + +function findRepositoryAnchor(projectRoot: string): CacheAnchorResolution { + const start = path.resolve(projectRoot); + let current = start; + while (true) { + const manifest = path.join(current, ".codegraph", "manifest.json"); + if (fs.existsSync(manifest) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { + return { anchor: current, layer: "manifest" }; + } + const gitPath = path.join(current, ".git"); + if (fs.existsSync(gitPath) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { + return { anchor: current, layer: "git" }; + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + return { anchor: start, layer: "project" }; +} + +function resolveCodegraphUserCacheRoot(): string { + const base = + process.platform === "win32" + ? process.env.LOCALAPPDATA?.trim() || path.join(os.homedir(), "AppData", "Local") + : process.env.XDG_CACHE_HOME?.trim() || path.join(os.homedir(), ".cache"); + return path.join(base, "codegraph"); +} + +export function projectCacheNamespace(projectRoot: string): string { + const rootIdentity = fileIdentityKey(path.resolve(projectRoot)); + const hash = crypto.createHash("sha256").update(rootIdentity).digest("hex"); + return `project-${hash}`; +} + +export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): CacheAnchorResolution { + const explicit = opts?.cacheDir?.trim(); + if (explicit) return { anchor: path.resolve(explicit), layer: "explicit" }; + const environment = process.env.CODEGRAPH_CACHE_DIR?.trim(); + if (environment) return { anchor: path.resolve(environment), layer: "environment" }; + const location = opts?.cacheLocation; + if (location === "project") return { anchor: path.resolve(projectRoot), layer: "project" }; + if (location === "user") return { anchor: resolveCodegraphUserCacheRoot(), layer: "user" }; + if (location && location !== "repo") return { anchor: path.resolve(location), layer: "explicit" }; + return findRepositoryAnchor(projectRoot); +} + +export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { + const root = path.resolve(projectRoot); + const resolution = resolveCacheAnchor(root, opts); + const anchor = isWritableDirectory(resolution.anchor) ? resolution.anchor : root; + const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); + if ( + sameRoot && + !opts?.cacheDir && + !process.env.CODEGRAPH_CACHE_DIR?.trim() && + (!opts?.cacheLocation || opts.cacheLocation === "project") + ) { + return path.join(root, ".codegraph-cache", "index-v1"); + } + const namespace = projectCacheNamespace(root); + const explicitBase = opts?.cacheDir?.trim() || process.env.CODEGRAPH_CACHE_DIR?.trim(); + if (explicitBase) { + const configured = path.resolve(explicitBase); + if (path.basename(configured) === namespace) return configured; + return path.join(configured, namespace); + } + const base = opts?.cacheLocation === "user" ? resolveCodegraphUserCacheRoot() : path.join(anchor, ".codegraph-cache"); + const candidate = path.join(path.resolve(base), "index-v1", namespace); + if (!sameRoot && !opts?.cacheLocation) { + const legacy = path.join(root, ".codegraph-cache", "index-v1"); + if (!fs.existsSync(candidate) && fs.existsSync(legacy)) return legacy; + } + return candidate; +} diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 40810896..bc0f385a 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -16,7 +16,8 @@ import { import { assertFilePathWithinRoot, fileIdentityKey, isFilePathWithinRoot } from "../../util/paths.js"; import { getGitBlobHashes } from "../../util/git.js"; import { stringifyUnknown } from "../../util/ast.js"; -import { cacheAbsolutePath, cacheRelativePath, cacheRoot, fileSignature } from "./module-cache.js"; +import { cacheAbsolutePath, cacheRelativePath, fileSignature } from "./module-cache.js"; +import { cacheRoot } from "./location.js"; import type { BuildOptions } from "../types.js"; import type { ManifestBuildOptions } from "./options.js"; diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index f9d84654..8f2ba761 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -2,7 +2,7 @@ import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } import crypto from "node:crypto"; import fs from "node:fs"; import fsp from "node:fs/promises"; -import os from "node:os"; + import path from "node:path"; import { supportForFile } from "../../languages.js"; import { getNativeRuntimeFingerprint } from "../../native/treeSitterNative.js"; @@ -26,6 +26,8 @@ import { } from "../../util/paths.js"; import { lruMapGet, lruMapSet } from "../../util/lruMap.js"; import { initCacheReport } from "./reports.js"; +import { cacheRoot } from "./location.js"; + import { getImplementationFingerprint } from "./options.js"; // v6: only reexports resolved inside the project are persisted as cache-relative paths. @@ -90,100 +92,7 @@ function reportMissingNodeSqlite(logLevel: import("../../logging.js").LogLevel | error, ); } -export function projectCacheNamespace(projectRoot: string): string { - const rootIdentity = fileIdentityKey(path.resolve(projectRoot)); - const hash = crypto.createHash("sha256").update(rootIdentity).digest("hex"); - return `project-${hash}`; -} - -export type CacheAnchorResolution = { - anchor: string; - layer: "explicit" | "environment" | "manifest" | "git" | "project" | "user"; -}; - -function isWritableDirectory(candidate: string): boolean { - try { - return fs.statSync(candidate).isDirectory() && fs.accessSync(candidate, fs.constants.W_OK) === undefined; - } catch { - return false; - } -} - -function isForbiddenAnchor(candidate: string): boolean { - const resolved = path.resolve(candidate); - const home = path.resolve(os.homedir()); - const parsed = path.parse(resolved); - return ( - fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root) - ); -} -function findRepositoryAnchor(projectRoot: string): CacheAnchorResolution { - const start = path.resolve(projectRoot); - let current = start; - while (true) { - const manifest = path.join(current, ".codegraph", "manifest.json"); - if (fs.existsSync(manifest) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { - return { anchor: current, layer: "manifest" }; - } - const gitPath = path.join(current, ".git"); - if (fs.existsSync(gitPath) && isWritableDirectory(current) && !isForbiddenAnchor(current)) { - return { anchor: current, layer: "git" }; - } - const parent = path.dirname(current); - if (parent === current) break; - current = parent; - } - return { anchor: start, layer: "project" }; -} - -function resolveCodegraphUserCacheRoot(): string { - const base = - process.platform === "win32" - ? process.env.LOCALAPPDATA?.trim() || path.join(os.homedir(), "AppData", "Local") - : process.env.XDG_CACHE_HOME?.trim() || path.join(os.homedir(), ".cache"); - return path.join(base, "codegraph"); -} -export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): CacheAnchorResolution { - const explicit = opts?.cacheDir?.trim(); - if (explicit) return { anchor: path.resolve(explicit), layer: "explicit" }; - const environment = process.env.CODEGRAPH_CACHE_DIR?.trim(); - if (environment) return { anchor: path.resolve(environment), layer: "environment" }; - const location = opts?.cacheLocation; - if (location === "project") return { anchor: path.resolve(projectRoot), layer: "project" }; - if (location === "user") return { anchor: resolveCodegraphUserCacheRoot(), layer: "user" }; - if (location && location !== "repo") return { anchor: path.resolve(location), layer: "explicit" }; - return findRepositoryAnchor(projectRoot); -} - -export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { - const root = path.resolve(projectRoot); - const resolution = resolveCacheAnchor(root, opts); - const anchor = isWritableDirectory(resolution.anchor) ? resolution.anchor : root; - const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); - if ( - sameRoot && - !opts?.cacheDir && - !process.env.CODEGRAPH_CACHE_DIR?.trim() && - (!opts?.cacheLocation || opts.cacheLocation === "project") - ) { - return path.join(root, ".codegraph-cache", "index-v1"); - } - const namespace = projectCacheNamespace(root); - const explicitBase = opts?.cacheDir?.trim() || process.env.CODEGRAPH_CACHE_DIR?.trim(); - if (explicitBase) { - const configured = path.resolve(explicitBase); - if (path.basename(configured) === namespace) return configured; - return path.join(configured, namespace); - } - const base = opts?.cacheLocation === "user" ? resolveCodegraphUserCacheRoot() : path.join(anchor, ".codegraph-cache"); - const candidate = path.join(path.resolve(base), "index-v1", namespace); - if (!sameRoot && !opts?.cacheLocation) { - const legacy = path.join(root, ".codegraph-cache", "index-v1"); - if (!fs.existsSync(candidate) && fs.existsSync(legacy)) return legacy; - } - return candidate; -} export function cacheRelativePath(projectRoot: string, file: string): string { const root = path.resolve(projectRoot); const absolute = path.isAbsolute(file) ? file : path.resolve(root, file); diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index b297a1e7..fdd90097 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -34,10 +34,10 @@ import { getImplementationFingerprint, normalizeGraphOptions } from "./options.j import { cacheAbsolutePath, cacheRelativePath, - cacheRoot, transformPersistedExportFromModule, type FileSignature, } from "./module-cache.js"; +import { cacheRoot } from "./location.js"; import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index f8d1df21..d0e3f315 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -74,7 +74,7 @@ import { type ManifestFileEntry, type PendingModuleCacheWrite, } from "./build-cache.js"; -import { cacheRoot } from "./build-cache/module-cache.js"; +import { cacheRoot } from "./build-cache/location.js"; import { type BuildOptions, type BuildReport, diff --git a/src/indexer/finalize.ts b/src/indexer/finalize.ts index fd565387..f39fcf36 100644 --- a/src/indexer/finalize.ts +++ b/src/indexer/finalize.ts @@ -7,7 +7,7 @@ import type { BloomFilterCache } from "../util/bloomFilter.js"; import type { ParsedFileContext } from "./parse-context.js"; import { retainedParsedCache } from "./parsed-cache.js"; import { buildReferenceCandidateIndex } from "./reference-candidates.js"; -import { cacheRoot } from "./build-cache/module-cache.js"; +import { cacheRoot } from "./build-cache/location.js"; import type { BuildOptions, BuildReport, ModuleIndex, ProjectIndex, ProjectIndexManifestEntry } from "./types.js"; export async function finalizeProjectIndex(args: { diff --git a/tests/cache-path-confinement.test.ts b/tests/cache-path-confinement.test.ts index f230c534..6f5fa045 100644 --- a/tests/cache-path-confinement.test.ts +++ b/tests/cache-path-confinement.test.ts @@ -16,9 +16,11 @@ import { } from "../src/indexer/build-cache/module-cache.js"; import { loadManifest } from "../src/indexer/build-cache/manifest.js"; import { + BLOOM_FILTER_SNAPSHOT_FILENAME, tryLoadPersistedBloomFilters, tryLoadProjectIndexSnapshot, } from "../src/indexer/build-cache/project-snapshot.js"; +import { cacheRoot } from "../src/indexer/build-cache/location.js"; import { isNativeTreeSitterAvailable } from "../src/native/treeSitterNative.js"; import { isNonNativeParserAvailable } from "../src/parserBackend.js"; import { mkTmpDir } from "./helpers/filesystem.js"; @@ -356,18 +358,17 @@ describe("persisted cache rehydration is confined to the project root", () => { await fsp.writeFile(path.join(root, "a.ts"), "export const a = 1;\n", "utf8"); await buildProjectIndex(root, { cache: "disk", threads: 1, useBloomFilters: true }); - const snapshotPath = snapshotPathFor(root); - const raw = await fsp.readFile(snapshotPath); + const bloomSnapshotPath = path.join(cacheRoot(root, { cache: "disk" }), BLOOM_FILTER_SNAPSHOT_FILENAME); + const raw = await fsp.readFile(bloomSnapshotPath); const payload = JSON.parse(brotliDecompressSync(raw).toString("utf8")) as { bloomFilters?: Record; }; const sampleFilter = Object.values(payload.bloomFilters ?? {})[0]; expect(sampleFilter).toBeDefined(); payload.bloomFilters = { "../../outside.ts": sampleFilter }; - await fsp.writeFile(snapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + await fsp.writeFile(bloomSnapshotPath, brotliCompressSync(Buffer.from(JSON.stringify(payload)))); + await fsp.rm(snapshotPathFor(root)); - // The bloom-only fast path reads this section directly, bypassing the whole-snapshot - // transform, so it must apply the same confinement check on its own before reuse. const bloomFilters = await tryLoadPersistedBloomFilters(root, { cache: "disk" }); expect(bloomFilters).toBeNull(); }); From edcf2aa980046a490dfc919287c33d4513128734 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 11:58:56 -0400 Subject: [PATCH 19/28] fix: preserve cache location defaults --- src/config.ts | 15 ++++++++++++-- tests/codegraph-config.test.ts | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/config.ts b/src/config.ts index 13dca7e9..8130c573 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,6 +20,16 @@ const stringArraySchema = z.array(z.string().trim().min(1)); const languageExtensionsSchema = z.record(z.string().trim().min(1), z.string().trim().min(1)); +const cacheLocationSchema = z + .string() + .trim() + .refine( + (location) => location === "project" || location === "repo" || location === "user" || path.isAbsolute(location), + { + message: 'Cache location must be "project", "repo", "user", or an absolute path.', + }, + ); + const codegraphConfigSchema = z .object({ discovery: z @@ -44,7 +54,7 @@ const codegraphConfigSchema = z .optional(), cache: z .object({ - location: z.string().trim().min(1), + location: cacheLocationSchema, }) .optional(), }) @@ -199,8 +209,9 @@ export async function loadCodegraphConfig(projectRoot: string): Promise { expect(config.discovery?.ignoreGlobs).toEqual(["tests/samples/**"]); }); + it("preserves repository cache anchoring when project config has no cache location", async () => { + const repositoryRoot = await mkRepo(); + const projectRoot = path.join(repositoryRoot, "packages", "app"); + await fs.mkdir(projectRoot, { recursive: true }); + await fs.writeFile(path.join(repositoryRoot, ".git"), "gitdir: external\n", "utf8"); + await writeConfig(projectRoot, {}); + vi.stubEnv("APPDATA", path.join(repositoryRoot, "missing-appdata")); + vi.stubEnv("XDG_CONFIG_HOME", path.join(repositoryRoot, "missing-xdg-config")); + + try { + const config = await loadCodegraphConfig(projectRoot); + const resolvedCacheRoot = cacheRoot(projectRoot, { + cache: "disk", + ...(config.cache ? { cacheLocation: config.cache.location } : {}), + }); + + expect(config.cache).toBeUndefined(); + expect(resolvedCacheRoot).not.toBe(path.join(projectRoot, ".codegraph-cache", "index-v1")); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("rejects relative cache locations and accepts absolute cache directories", async () => { + const root = await mkRepo(); + await writeConfig(root, { cache: { location: "relative-cache" } }); + + await expect(loadCodegraphConfig(root)).rejects.toThrow(/Invalid codegraph\.config\.json/); + + const absoluteCache = path.join(root, "cache"); + await writeConfig(root, { cache: { location: absoluteCache } }); + + await expect(loadCodegraphConfig(root)).resolves.toEqual({ cache: { location: absoluteCache } }); + }); + it("merges config discovery with explicit discovery overrides", () => { const merged = mergeDiscoveryOptions( { From 19da28da08c5de7fd8326291072b4819ea3afb70 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 16:23:50 -0400 Subject: [PATCH 20/28] fix: preserve configured cache locations --- docs/cli.md | 1 + src/cli/help.ts | 2 +- src/duplicates/unitCache.ts | 1 - src/duplicates/units.ts | 4 ++-- src/indexer/build-cache/location.ts | 2 +- src/indexer/navigation.ts | 3 ++- tests/cli-command-modules.test.ts | 12 +++++++++++- tests/codegraph-config.test.ts | 12 +++++++++++- tests/parsed-cache-eviction.test.ts | 2 +- 9 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 8758f4a7..96932d49 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -21,6 +21,7 @@ The `graph` command without output-format flags writes Mermaid to stdout. Use `- Numeric options such as `--limit`, `--threads`, `--depth`, `--max-refs`, and token bounds must be integers in their documented ranges; invalid numeric values fail instead of being silently clamped or ignored. Default workflow: + ## Cache location Index caches store project-relative paths, so a cache can be moved with its project. Cache selection precedence is `--cache-dir`, `CODEGRAPH_CACHE_DIR`, `cache.location` in project config (then user config), repository metadata, then the project root. `cache.location` accepts `project`, `repo`, `user`, or an absolute path; `--root` remains the project scope boundary. diff --git a/src/cli/help.ts b/src/cli/help.ts index 22c275b2..062640e7 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -260,7 +260,7 @@ A bare positional is a text regex. Use --query explicitly for Tree-sitter querie export const SQL_HELP_TEXT = `codegraph sql - Query a graph SQLite export read-only Usage: codegraph sql "SELECT ..." [--json | --pretty] - codegraph sql --db --query "SELECT ..." [--json] + codegraph sql --db --query "SELECT ..." [--json | --pretty] `; export const SYMBOLS_HELP_TEXT = `codegraph symbols - Deterministic workspace-symbol lookup diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index d768d04c..51c62d59 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -367,7 +367,6 @@ export function tryLoadDuplicateUnitsFromCache( } catch { return null; } - } export type PendingDuplicateUnitCacheWrite = { diff --git a/src/duplicates/units.ts b/src/duplicates/units.ts index 71cac385..25c501fb 100644 --- a/src/duplicates/units.ts +++ b/src/duplicates/units.ts @@ -128,7 +128,7 @@ function formatDuplicateChunkHandle(file: string, line: number): string { return ["chunk", encodeURIComponent(file), String(line)].join(":"); } -function formatDuplicateSqlHandle(file: string, name: string, line: number): string { +export function formatDuplicateSqlHandle(file: string, name: string, line: number): string { return ["sql", encodeURIComponent(name), encodeURIComponent(file), String(line)].join(":"); } @@ -140,7 +140,7 @@ function sqlHandleForDuplicateSymbol(symbol: SymbolDef, file: string): string | return formatDuplicateSqlHandle(file, symbol.localName, symbol.range.start.line); } -function formatDuplicateSymbolHandle(file: string, name: string, line: number, column: number): string { +export function formatDuplicateSymbolHandle(file: string, name: string, line: number, column: number): string { return ["symbol", encodeURIComponent(file), encodeURIComponent(name), String(line), String(column)].join(":"); } diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts index 69a34f3c..7a355f3f 100644 --- a/src/indexer/build-cache/location.ts +++ b/src/indexer/build-cache/location.ts @@ -76,7 +76,7 @@ export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): Ca export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { const root = path.resolve(projectRoot); const resolution = resolveCacheAnchor(root, opts); - const anchor = isWritableDirectory(resolution.anchor) ? resolution.anchor : root; + const anchor = resolution.layer === "explicit" || isWritableDirectory(resolution.anchor) ? resolution.anchor : root; const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); if ( sameRoot && diff --git a/src/indexer/navigation.ts b/src/indexer/navigation.ts index 57aae29d..739b9ae9 100644 --- a/src/indexer/navigation.ts +++ b/src/indexer/navigation.ts @@ -64,7 +64,8 @@ export async function goToDefinition( if (sqlResult) return sqlResult; const context = - parsedContext ?? (await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), index.languageExtensions)); + parsedContext ?? + (await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), index.languageExtensions)); const sup = context.sup; const lang = context.lang; const source = context.source; diff --git a/tests/cli-command-modules.test.ts b/tests/cli-command-modules.test.ts index 15cd9387..52ce4d1b 100644 --- a/tests/cli-command-modules.test.ts +++ b/tests/cli-command-modules.test.ts @@ -11,7 +11,13 @@ import { buildDoctorReport, findStaleNpmRetirementPaths } from "../src/cli/docto import { handleGraphCommand, type GraphCommandContext } from "../src/cli/graph.js"; import { handleGraphDeltaCommand } from "../src/cli/graphDelta.js"; import { handleGraphQueryCommand, type GraphQueryCommandContext } from "../src/cli/graphQueries.js"; -import { CLI_HELP_TEXT, FILE_HELP_TEXT, MCP_SERVE_HELP_TEXT, PACKET_HELP_TEXT } from "../src/cli/help.js"; +import { + CLI_HELP_TEXT, + FILE_HELP_TEXT, + MCP_SERVE_HELP_TEXT, + PACKET_HELP_TEXT, + SQL_HELP_TEXT, +} from "../src/cli/help.js"; import { handleImpactCommand, type ImpactCommandContext } from "../src/cli/impact.js"; import { handleIndexCommand, type IndexCommandContext } from "../src/cli/index.js"; import { handleHotspotsCommand, handleInspectCommand, type InspectCommandContext } from "../src/cli/inspect.js"; @@ -332,6 +338,10 @@ describe("CLI command modules", () => { expect(commands).toContain("Serve MCP tools for agent graph navigation"); }); + test("documents --pretty for both SQL command forms", () => { + expect(SQL_HELP_TEXT).toContain('codegraph sql --db --query "SELECT ..." [--json | --pretty]'); + }); + test("lists all public top-level commands in CLI help", () => { const commands = CLI_HELP_TEXT.slice(CLI_HELP_TEXT.indexOf("Commands:"), CLI_HELP_TEXT.indexOf("Graph Options:")); diff --git a/tests/codegraph-config.test.ts b/tests/codegraph-config.test.ts index 6d49fc16..41d37c1a 100644 --- a/tests/codegraph-config.test.ts +++ b/tests/codegraph-config.test.ts @@ -6,7 +6,7 @@ import { hasDiscoveryOptions, loadCodegraphConfig, mergeDiscoveryOptions, mergeG import { searchCodegraph } from "../src/agent/search.js"; import { buildProjectIndex, type BuildReport } from "../src/indexer/build-index.js"; import { diffBuildOptions, summarizeBuildOptions } from "../src/indexer/build-cache.js"; -import { cacheRoot } from "../src/indexer/build-cache/location.js"; +import { cacheRoot, projectCacheNamespace } from "../src/indexer/build-cache/location.js"; import { normalizeLanguageExtensions, supportForFile } from "../src/languages.js"; import { fileIdentityKey } from "../src/util/paths.js"; import { runTsxScriptOrThrow } from "./helpers/cli.js"; @@ -101,6 +101,16 @@ describe("codegraph config", () => { await expect(loadCodegraphConfig(root)).resolves.toEqual({ cache: { location: absoluteCache } }); }); + it("preserves an explicit cache location before its directory exists", async () => { + const root = await mkRepo(); + const location = path.join(root, "new-cache-anchor"); + + expect(cacheRoot(root, { cache: "disk", cacheLocation: location })).toBe( + path.join(location, ".codegraph-cache", "index-v1", projectCacheNamespace(root)), + ); + await expect(fs.stat(location)).rejects.toThrow(); + }); + it("merges config discovery with explicit discovery overrides", () => { const merged = mergeDiscoveryOptions( { diff --git a/tests/parsed-cache-eviction.test.ts b/tests/parsed-cache-eviction.test.ts index fee10b20..73af15ec 100644 --- a/tests/parsed-cache-eviction.test.ts +++ b/tests/parsed-cache-eviction.test.ts @@ -22,7 +22,7 @@ describe("parsed AST cache eviction", () => { parsedCacheMaxEntries: 2, }); - expect(index.parsed).toHaveLength(2); + expect(index.parsed?.size).toBe(2); } finally { await fsp.rm(root, { recursive: true, force: true }); } From 925cbc4328db762950acb536418b11a084fced27 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 16:59:07 -0400 Subject: [PATCH 21/28] fix(doctor): report the effective cache anchor/layer, not the intended one buildDoctorReport() called resolveCacheAnchor() and cacheRoot() independently, so when cacheRoot() falls back to the project root (non-writable anchor) or reuses a legacy in-project cache, the reported anchor/layer disagreed with the actual cache path. Added resolveCacheLocation() as the single source of truth returning {path, anchor, layer} together; cacheRoot() now derives from it and doctor uses it directly. --- src/cli/doctor.ts | 6 +++--- src/indexer/build-cache.ts | 2 +- src/indexer/build-cache/location.ts | 30 ++++++++++++++++++++++------- tests/cache-invalidation.test.ts | 15 +++++++++++++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 5abfaa2b..2f37a79b 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import { cacheRoot, resolveCacheAnchor } from "../indexer/build-cache/location.js"; +import { resolveCacheLocation } from "../indexer/build-cache/location.js"; import { isNativeTreeSitterAvailable, getNativeBindingOrigin, @@ -315,11 +315,11 @@ export function buildDoctorReport(indexPath?: string): DoctorReport { const origin = getNativeBindingOrigin(); const runtimeIdentity = captureCodegraphRuntimeIdentity(origin); const update = createInstalledVersionChecker(runtimeIdentity, { warn: () => undefined }).check(true); - const cacheResolution = resolveCacheAnchor(process.cwd()); + const cacheResolution = resolveCacheLocation(process.cwd()); return { package: packageIdentity, cache: { - path: normalizePathForDisplay(cacheRoot(process.cwd())), + path: normalizePathForDisplay(cacheResolution.path), anchor: normalizePathForDisplay(cacheResolution.anchor), layer: cacheResolution.layer, }, diff --git a/src/indexer/build-cache.ts b/src/indexer/build-cache.ts index ddaaf5b7..637c357c 100644 --- a/src/indexer/build-cache.ts +++ b/src/indexer/build-cache.ts @@ -25,7 +25,7 @@ export { type FileSignature, type PendingModuleCacheWrite, } from "./build-cache/module-cache.js"; -export { cacheRoot } from "./build-cache/location.js"; +export { cacheRoot, resolveCacheLocation } from "./build-cache/location.js"; export { BLOOM_FILTER_SNAPSHOT_FILENAME, BLOOM_FILTER_SNAPSHOT_VERSION, diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts index 7a355f3f..6fc344bf 100644 --- a/src/indexer/build-cache/location.ts +++ b/src/indexer/build-cache/location.ts @@ -73,10 +73,20 @@ export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): Ca return findRepositoryAnchor(projectRoot); } -export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { +export type CacheLocationResolution = CacheAnchorResolution & { path: string }; + +/** + * Resolves the effective on-disk cache path along with the anchor/layer that actually + * produced it. This can differ from `resolveCacheAnchor`'s intended anchor when that + * anchor is not writable (falls back to the project root) or when an existing legacy + * in-project cache is reused instead of the configured anchor. + */ +export function resolveCacheLocation(projectRoot: string, opts?: BuildOptions): CacheLocationResolution { const root = path.resolve(projectRoot); const resolution = resolveCacheAnchor(root, opts); - const anchor = resolution.layer === "explicit" || isWritableDirectory(resolution.anchor) ? resolution.anchor : root; + const anchorWritable = resolution.layer === "explicit" || isWritableDirectory(resolution.anchor); + const anchor = anchorWritable ? resolution.anchor : root; + const effectiveLayer = anchorWritable ? resolution.layer : "project"; const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); if ( sameRoot && @@ -84,20 +94,26 @@ export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { !process.env.CODEGRAPH_CACHE_DIR?.trim() && (!opts?.cacheLocation || opts.cacheLocation === "project") ) { - return path.join(root, ".codegraph-cache", "index-v1"); + return { path: path.join(root, ".codegraph-cache", "index-v1"), anchor, layer: effectiveLayer }; } const namespace = projectCacheNamespace(root); const explicitBase = opts?.cacheDir?.trim() || process.env.CODEGRAPH_CACHE_DIR?.trim(); if (explicitBase) { const configured = path.resolve(explicitBase); - if (path.basename(configured) === namespace) return configured; - return path.join(configured, namespace); + const configuredPath = path.basename(configured) === namespace ? configured : path.join(configured, namespace); + return { path: configuredPath, anchor, layer: effectiveLayer }; } const base = opts?.cacheLocation === "user" ? resolveCodegraphUserCacheRoot() : path.join(anchor, ".codegraph-cache"); const candidate = path.join(path.resolve(base), "index-v1", namespace); if (!sameRoot && !opts?.cacheLocation) { const legacy = path.join(root, ".codegraph-cache", "index-v1"); - if (!fs.existsSync(candidate) && fs.existsSync(legacy)) return legacy; + if (!fs.existsSync(candidate) && fs.existsSync(legacy)) { + return { path: legacy, anchor: root, layer: "project" }; + } } - return candidate; + return { path: candidate, anchor, layer: effectiveLayer }; +} + +export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { + return resolveCacheLocation(projectRoot, opts).path; } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 7adf678a..8863ccae 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -2545,4 +2545,19 @@ describe("Cache invalidation and strict hashing", () => { expect(cachePath).not.toBe(path.join(projectRoot, ".codegraph-cache", "index-v1")); expect(cachePath).not.toBe(siblingCachePath); }); + + it("reports the legacy in-project anchor and layer when reusing a legacy cache under a git anchor", async () => { + const repoRoot = await mkTmpDir("dg-cache-legacy-anchor-repo-"); + const projectRoot = path.join(repoRoot, "packages", "app"); + await fsp.mkdir(projectRoot, { recursive: true }); + await fsp.writeFile(path.join(repoRoot, ".git"), "gitdir: external\n", "utf8"); + const legacyCachePath = path.join(projectRoot, ".codegraph-cache", "index-v1"); + await fsp.mkdir(legacyCachePath, { recursive: true }); + + const resolution = buildCache.resolveCacheLocation(projectRoot, { cache: "disk" }); + + expect(resolution.path).toBe(legacyCachePath); + expect(fileIdentityKey(resolution.anchor)).toBe(fileIdentityKey(projectRoot)); + expect(resolution.layer).toBe("project"); + }); }); From 275d6a1bbf559cedcad1ff3b383f51d3c5076414 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 17:46:24 -0400 Subject: [PATCH 22/28] fix: address suppressed PR review findings across cache portability, CLI, and session config - project-snapshot.ts: guard legacy v4/v5 snapshot migration against the missing fileSignatures field (previously threw, silently forcing a full rebuild via the outer catch); drop the redundant projectRootMatches root-equality gate from per-file module reuse and both bloom-filter validators now that transformPath already confines rehydrated paths to the current root, so portable current-version snapshots survive a project move. - location.ts: derive the repo-anchored cache namespace from the path relative to the resolved anchor (not the absolute project root), so a moved repository keeps discovering its subproject caches; explicit/user cache locations keep absolute-path hashing since they don't travel with the project. - doctor.ts: add a Cache section (path/anchor/layer) to the pretty formatter, matching the JSON report shape. - docs/cli.md: move the Cache location section after the Default workflow bullet list instead of splitting it. - query-index/store.ts: bound candidateChunksForTerms' SQL prefetch with a LIMIT across path batches, restoring the overfetch-then-cap strategy the replaced FTS helpers had (previously unbounded, so a common term could materialize every matching chunk before the in-memory cap). - cli/{navigation,graph,impact,review,graphDelta,index,commandTable}.ts: thread --cache-dir and config cache.location through every build-option path that constructs its own BuildOptions instead of buildAgentOptions(), so those commands stop silently ignoring both. - session.ts: merge codegraph.config.json's cache.location into currentBuildOptions() (matching discovery/graph/languageExtensions), and include cacheLocation in session identity normalization so sessions with different explicit cache locations don't collide. Adds regression coverage for each fix in cache-invalidation.test.ts, query-index.test.ts, session.test.ts, cli-regressions.test.ts, and cli-command-modules.test.ts. --- docs/cli.md | 8 +-- src/agent/query-index/store.ts | 12 +++- src/cli/commandTable.ts | 9 +++ src/cli/doctor.ts | 13 ++++- src/cli/graph.ts | 11 +++- src/cli/graphDelta.ts | 6 +- src/cli/impact.ts | 9 ++- src/cli/index.ts | 6 +- src/cli/navigation.ts | 5 ++ src/cli/review.ts | 6 +- src/indexer/build-cache/location.ts | 12 ++-- src/indexer/build-cache/project-snapshot.ts | 9 +-- src/session.ts | 11 +++- tests/cache-invalidation.test.ts | 63 +++++++++++++++++++++ tests/cli-command-modules.test.ts | 4 ++ tests/cli-regressions.test.ts | 30 ++++++++++ tests/query-index.test.ts | 24 ++++++++ tests/session.test.ts | 41 ++++++++++++++ 18 files changed, 254 insertions(+), 25 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 96932d49..f807f356 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -22,16 +22,16 @@ Numeric options such as `--limit`, `--threads`, `--depth`, `--max-refs`, and tok Default workflow: -## Cache location - -Index caches store project-relative paths, so a cache can be moved with its project. Cache selection precedence is `--cache-dir`, `CODEGRAPH_CACHE_DIR`, `cache.location` in project config (then user config), repository metadata, then the project root. `cache.location` accepts `project`, `repo`, `user`, or an absolute path; `--root` remains the project scope boundary. - - code review: `codegraph review` - blast-radius follow-up: `codegraph impact --base HEAD --head WORKTREE` - unfamiliar repo: `codegraph explore "how does auth reach db?" --root .` - first-turn map: `codegraph orient --root . --budget small` - targeted follow-up: `codegraph search "" --json` then `codegraph explain ` +## Cache location + +Index caches store project-relative paths, so a cache can be moved with its project. Cache selection precedence is `--cache-dir`, `CODEGRAPH_CACHE_DIR`, `cache.location` in project config (then user config), repository metadata, then the project root. `cache.location` accepts `project`, `repo`, `user`, or an absolute path; `--root` remains the project scope boundary. + ## Runtime selection The CLI defaults to `--native auto`, which uses the native Tree-sitter path when a compatible native artifact is available and falls back automatically otherwise. diff --git a/src/agent/query-index/store.ts b/src/agent/query-index/store.ts index aa44f31f..773d81a7 100644 --- a/src/agent/query-index/store.ts +++ b/src/agent/query-index/store.ts @@ -322,8 +322,13 @@ export class QueryIndexStore { } } - candidateChunksForTerms(terms: readonly string[], paths: readonly string[]): StoredQueryIndexChunk[] { + candidateChunksForTerms( + terms: readonly string[], + paths: readonly string[], + limit = QUERY_INDEX_CANDIDATE_PREFETCH_LIMIT, + ): StoredQueryIndexChunk[] { if (!terms.length || !paths.length) return []; + const normalizedLimit = normalizedCandidateLimit(limit); const ftsTerms = terms.filter((term) => codePointLength(term) >= 3); const directTerms = terms.filter((term) => codePointLength(term) < 3); const conditions: string[] = []; @@ -345,9 +350,10 @@ export class QueryIndexStore { : ""; const candidates = new Map(); const batchSize = 500; - for (let offset = 0; offset < paths.length; offset += batchSize) { + for (let offset = 0; offset < paths.length && candidates.size < normalizedLimit; offset += batchSize) { const batch = paths.slice(offset, offset + batchSize); const placeholders = batch.map(() => "?").join(", "); + const remaining = normalizedLimit - candidates.size; const rows = this.db .prepare( ` @@ -359,10 +365,12 @@ export class QueryIndexStore { WHERE files.path IN (${placeholders}) AND (${conditions.join(" OR ")}) ORDER BY files.path, chunks.ordinal + LIMIT ? `, ) .all( ...(ftsTerms.length ? [ftsQuery, ...batch, ...directParameters] : [...batch, ...directParameters]), + remaining, ) as Array>; for (const row of rows) { const chunk = storedCandidateChunkFromRow(row); diff --git a/src/cli/commandTable.ts b/src/cli/commandTable.ts index ae62cdd8..6188b6e7 100644 --- a/src/cli/commandTable.ts +++ b/src/cli/commandTable.ts @@ -326,6 +326,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { projectRootFs: ctx.projectRootFs, files, languageExtensions: ctx.config.languages?.extensions, + cacheLocation: ctx.config.cache?.location, getOpt: ctx.getOpt, hasFlag: ctx.hasFlag, cwd: getCwd, @@ -357,6 +358,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { await handleGraphCommand({ projectRootFs: ctx.projectRootFs, discoveryOptions: ctx.discoveryOptions, + cacheLocation: ctx.config.cache?.location, nativeMode: ctx.nativeMode, workerOpts: ctx.workerOpts, progressHandler: ctx.progressHandler, @@ -392,6 +394,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { nativeMode: ctx.nativeMode, workerOpts: ctx.workerOpts, languageExtensions: ctx.config.languages?.extensions, + cacheLocation: ctx.config.cache?.location, progressHandler: ctx.progressHandler, graphOptions: ctx.hasGraphOverrides || ctx.nativeMode !== "auto" ? ctx.buildGraphOptions() : undefined, reportEnabled: ctx.reportEnabled, @@ -413,6 +416,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { run: async (ctx) => { const driftGraphOptions = ctx.hasGraphOverrides || ctx.nativeMode !== "auto" ? ctx.buildGraphOptions() : undefined; + const driftCacheDir = ctx.getOpt("--cache-dir"); const { handleDriftCommand } = await import("./drift.js"); await handleDriftCommand({ projectRootFs: ctx.projectRootFs, @@ -427,6 +431,8 @@ export const CLI_COMMAND_TABLE: Readonly> = { ...(ctx.nativeMode !== "auto" ? { native: ctx.nativeMode } : {}), ...(ctx.config.languages?.extensions ? { languageExtensions: ctx.config.languages.extensions } : {}), ...ctx.workerOpts, + ...(driftCacheDir ? { cacheDir: driftCacheDir } : {}), + ...(ctx.config.cache?.location ? { cacheLocation: ctx.config.cache.location } : {}), }, writeJSONLine, writeStdoutLine, @@ -503,6 +509,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { await handleImpactCommand({ projectRootFs: ctx.projectRootFs, discoveryOptions: ctx.discoveryOptions, + cacheLocation: ctx.config.cache?.location, getOpt: ctx.getOpt, hasFlag: ctx.hasFlag, parsedOptions: ctx.parsed.options, @@ -555,6 +562,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { await handleReviewCommand({ projectRootFs: ctx.projectRootFs, discoveryOptions: ctx.discoveryOptions, + cacheLocation: ctx.config.cache?.location, reportFile: ctx.reportFile, commandReport, getOpt: ctx.getOpt, @@ -724,6 +732,7 @@ function navigationArgs(ctx: CliProjectContext) { nativeMode: ctx.nativeMode, workerOpts: ctx.workerOpts, progressHandler: ctx.progressHandler, + cacheLocation: ctx.config.cache?.location, writeJSONLine, writeStdoutLine, writeStderrLine, diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 2f37a79b..59517bb5 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -210,7 +210,7 @@ function pushNestedSection(body: string[], title: string, nestedBody: readonly s } } -/** Pretty formatter for `codegraph doctor`. Stable Package/Native(+Origin/Update) nesting matches JSON. */ +/** Pretty formatter for `codegraph doctor`. Stable Package/Cache/Native(+Origin/Update) nesting matches JSON. */ export function formatDoctorSummary(report: DoctorReport): string { const lines: string[] = []; const pkg = asRecord(report.package); @@ -224,6 +224,17 @@ export function formatDoctorSummary(report: DoctorReport): string { ]), ); + const cache = asRecord(report.cache); + pushSection( + lines, + "Cache", + formatLabeledFields([ + ["Path", readField(cache, "path")], + ["Anchor", readField(cache, "anchor")], + ["Layer", readField(cache, "layer")], + ]), + ); + const native = asRecord(report.native); const supportedLanguageIds = readField(native, "supportedLanguageIds"); const nativeBody = formatLabeledFields([ diff --git a/src/cli/graph.ts b/src/cli/graph.ts index d6475a9c..e5c564dc 100644 --- a/src/cli/graph.ts +++ b/src/cli/graph.ts @@ -13,7 +13,7 @@ import { graphToMermaidSymbolsWithFiles, } from "../graphs/symbol-render.js"; import { buildProjectIndexFromFiles, buildProjectIndexIncremental } from "../indexer/build-index.js"; -import { type BuildOptions, type BuildReport } from "../indexer/types.js"; +import { type BuildOptions, type BuildReport, type CacheLocation } from "../indexer/types.js"; import { summarizeAnalysis } from "../analysisSummary.js"; import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; import { updateGraphSqlite, writeGraphSqlite } from "../sqlite.js"; @@ -65,6 +65,7 @@ export type GraphCommandContext = { nativeMode: NativeRuntimeMode; workerOpts: { useNativeWorkers: true } | Record; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; graphFlags: { fast: boolean; resolveNodeModules: boolean; @@ -247,6 +248,11 @@ export async function handleGraphCommand(context: GraphCommandContext): Promise< const threads = parseNonNegativeIntegerOption(context.getOpt("--threads"), "--threads", 0); const cache = parseCacheModeOption(context.getOpt("--cache")); const cacheStrict = context.hasFlag("--cache-strict"); + const cacheDir = context.getOpt("--cache-dir"); + const cacheDirOptions: Pick = { + ...(cacheDir ? { cacheDir } : {}), + ...(context.cacheLocation ? { cacheLocation: context.cacheLocation } : {}), + }; const stable = context.hasFlag("--stable"); let format: "mermaid" | "dot" | "json" = "mermaid"; if (context.hasFlag("--json")) { @@ -305,6 +311,7 @@ export async function handleGraphCommand(context: GraphCommandContext): Promise< discovery: context.discoveryOptions, ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), ...context.workerOpts, + ...cacheDirOptions, ...(sqliteCacheMode !== undefined ? { cache: sqliteCacheMode } : {}), cacheStrict, files: changedSet.existingFiles, @@ -320,6 +327,7 @@ export async function handleGraphCommand(context: GraphCommandContext): Promise< discovery: context.discoveryOptions, ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), ...context.workerOpts, + ...cacheDirOptions, ...(sqliteCacheMode !== undefined ? { cache: sqliteCacheMode } : {}), cacheStrict, graph: graphOptions, @@ -367,6 +375,7 @@ export async function handleGraphCommand(context: GraphCommandContext): Promise< discovery: context.discoveryOptions, ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), ...context.workerOpts, + ...cacheDirOptions, cache: cache ?? "disk", cacheStrict, graph: { diff --git a/src/cli/graphDelta.ts b/src/cli/graphDelta.ts index ea8c0890..1148d901 100644 --- a/src/cli/graphDelta.ts +++ b/src/cli/graphDelta.ts @@ -1,6 +1,6 @@ import fsp from "node:fs/promises"; import { buildGraphDelta } from "../indexer/build-index.js"; -import { type IncrementalBuildOptions } from "../indexer/types.js"; +import { type CacheLocation, type IncrementalBuildOptions } from "../indexer/types.js"; import { type GraphBuildOptions } from "../graphs/types.js"; import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; import { normalizePath, resolveFilePathFromRoot } from "../util/paths.js"; @@ -18,6 +18,7 @@ export type GraphDeltaCommandContext = { workerOpts: { useNativeWorkers: true } | Record; graphOptions: GraphBuildOptions | undefined; languageExtensions: IncrementalBuildOptions["languageExtensions"]; + cacheLocation: CacheLocation | undefined; gitBase: string | undefined; gitHead: string | undefined; changedSince: string | undefined; @@ -72,6 +73,9 @@ export async function handleGraphDeltaCommand(context: GraphDeltaCommandContext) if (context.languageExtensions) deltaOptions.languageExtensions = context.languageExtensions; if (context.nativeMode !== "auto") deltaOptions.native = context.nativeMode; if (cache !== undefined) deltaOptions.cache = cache; + const cacheDir = context.getOpt("--cache-dir"); + if (cacheDir) deltaOptions.cacheDir = cacheDir; + if (context.cacheLocation) deltaOptions.cacheLocation = context.cacheLocation; if (context.gitBase) deltaOptions.gitBase = context.gitBase; if (context.gitHead) deltaOptions.gitHead = context.gitHead; if (context.changedSince) deltaOptions.changedSince = context.changedSince; diff --git a/src/cli/impact.ts b/src/cli/impact.ts index 7a023eb2..3490a6fa 100644 --- a/src/cli/impact.ts +++ b/src/cli/impact.ts @@ -1,5 +1,5 @@ import { loadCurrentProjectIndex, type LoadCurrentProjectIndexOptions } from "../indexer/load-current-index.js"; -import type { BuildOptions, BuildReport, ProjectIndex } from "../indexer/types.js"; +import type { BuildOptions, BuildReport, CacheLocation, ProjectIndex } from "../indexer/types.js"; import { analyzeImpactFromDiff, type ChangedSymbol, @@ -45,6 +45,7 @@ type ImpactOptionsBuilder = Partial & { diffText?: string; threads?: number; cache?: BuildOptions["cache"]; + cacheDir?: string; cacheStrict?: boolean; cacheVerify?: boolean; }; @@ -59,6 +60,7 @@ export type ImpactCommandContext = { workerOpts: { useNativeWorkers: true } | Record; graphOptions: GraphBuildOptions | undefined; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; readStdin: () => Promise; writeJSONLine: (value: unknown) => void; writeStdoutLine: (message: string) => void; @@ -333,6 +335,9 @@ function applyAnalysisOptions(context: ImpactCommandContext, options: ImpactOpti const cache = parseCacheModeOption(context.getOpt("--cache")); if (cache !== undefined) options.cache = cache; + const cacheDir = context.getOpt("--cache-dir"); + if (cacheDir) options.cacheDir = cacheDir; + if (context.hasFlag("--cache-strict")) options.cacheStrict = true; if (context.hasFlag("--cache-verify")) options.cacheVerify = true; if (context.hasFlag("--compact")) options.compact = true; @@ -438,6 +443,8 @@ function buildIndexOptions( ...(keepParsed ? { keepParsed } : {}), ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), ...context.workerOpts, + ...(options.cacheDir ? { cacheDir: options.cacheDir } : {}), + ...(context.cacheLocation ? { cacheLocation: context.cacheLocation } : {}), ...(options.cacheStrict ? { cacheStrict: true } : {}), ...(options.cacheVerify ? { cacheVerify: true } : {}), }; diff --git a/src/cli/index.ts b/src/cli/index.ts index c0329e0d..7c01c522 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,6 +1,6 @@ import { performance } from "node:perf_hooks"; import { buildProjectIndexFromFiles, buildProjectIndexIncremental } from "../indexer/build-index.js"; -import { type BuildOptions, type BuildReport } from "../indexer/types.js"; +import { type BuildOptions, type BuildReport, type CacheLocation } from "../indexer/types.js"; import { summarizeAnalysis, type AnalysisSummary } from "../analysisSummary.js"; import { type GraphBuildOptions } from "../graphs/types.js"; import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; @@ -60,6 +60,7 @@ export type IndexCommandContext = { languageExtensions: LanguageExtensionMap | undefined; workerOpts: { useNativeWorkers: true } | Record; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; graphOptions: GraphBuildOptions | undefined; reportEnabled: boolean; reportFile: string | undefined; @@ -87,6 +88,7 @@ export async function handleIndexCommand(context: IndexCommandContext): Promise< } const threads = parseNonNegativeIntegerOption(context.getOpt("--threads"), "--threads", 0); const cache = parseCacheModeOption(context.getOpt("--cache")); + const cacheDir = context.getOpt("--cache-dir"); const cacheStrict = context.hasFlag("--cache-strict"); const full = context.hasFlag("--json") || context.hasFlag("--full"); const cacheVerify = context.hasFlag("--cache-verify"); @@ -108,6 +110,8 @@ export async function handleIndexCommand(context: IndexCommandContext): Promise< ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), ...(context.languageExtensions ? { languageExtensions: context.languageExtensions } : {}), ...context.workerOpts, + ...(cacheDir ? { cacheDir } : {}), + ...(context.cacheLocation ? { cacheLocation: context.cacheLocation } : {}), cache: cache ?? "disk", cacheStrict, cacheVerify, diff --git a/src/cli/navigation.ts b/src/cli/navigation.ts index aed808b6..2dbe8563 100644 --- a/src/cli/navigation.ts +++ b/src/cli/navigation.ts @@ -5,6 +5,7 @@ import { parseAgentSymbolHandle } from "../agent/handles.js"; import { SymbolKind, type BuildOptions, + type CacheLocation, type FindReferencesResult, type GoToResult, type ModuleIndex, @@ -28,6 +29,7 @@ export type NavigationCommandContext = { nativeMode: NativeRuntimeMode; workerOpts: { useNativeWorkers: true } | Record; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; writeJSONLine: (value: unknown) => void; writeStdoutLine: (message: string) => void; writeStderrLine: (message: string) => void; @@ -38,10 +40,13 @@ export type NavigationCommandContext = { // inputs are unchanged. Pass --cache off to opt out of persisted reuse for one invocation. function indexOptions(context: NavigationCommandContext): LoadCurrentProjectIndexOptions { const cache = parseCacheModeOption(context.getOpt("--cache")); + const cacheDir = context.getOpt("--cache-dir"); return { onProgress: context.progressHandler, discovery: context.discoveryOptions, ...(cache ? { cache } : {}), + ...(cacheDir ? { cacheDir } : {}), + ...(context.cacheLocation ? { cacheLocation: context.cacheLocation } : {}), ...(context.hasFlag("--cache-strict") ? { cacheStrict: true } : {}), ...(context.hasFlag("--cache-verify") ? { cacheVerify: true } : {}), ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), diff --git a/src/cli/review.ts b/src/cli/review.ts index b57d9089..4d6aba7c 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -4,7 +4,7 @@ import { formatMarkdownLinkCheckResult } from "./links.js"; import { buildReviewReport, type ReviewBuildReport, type ReviewDepth, type ReviewReport } from "../review.js"; import type { CandidateTestFile } from "../impact/context.js"; import { formatRequiredArgumentCount } from "../impact/reportShared.js"; -import type { BuildOptions, BuildReport, ProjectIndex } from "../indexer/types.js"; +import type { BuildOptions, BuildReport, CacheLocation, ProjectIndex } from "../indexer/types.js"; import { type GraphBuildOptions } from "../graphs/types.js"; import { appendDuplicateLeadSummary, @@ -49,6 +49,7 @@ export type ReviewCommandContext = { useNativeWorkers: boolean; graphOptions: GraphBuildOptions | undefined; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; writeJSONLine: (value: unknown) => void; writeStdoutLine: (message: string) => void; writeStderrLine: (message: string) => void; @@ -315,6 +316,9 @@ export async function handleReviewCommand(context: ReviewCommandContext): Promis if (context.useNativeWorkers) reviewOpts.useNativeWorkers = true; if (cacheStrict) reviewOpts.cacheStrict = true; if (cacheVerify) reviewOpts.cacheVerify = true; + const cacheDir = context.getOpt("--cache-dir"); + if (cacheDir) reviewOpts.cacheDir = cacheDir; + if (context.cacheLocation) reviewOpts.cacheLocation = context.cacheLocation; if (incrementalStrict) reviewOpts.incrementalStrict = true; if (context.graphOptions) reviewOpts.graph = context.graphOptions; if (context.progressHandler) reviewOpts.onProgress = context.progressHandler; diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts index 6fc344bf..a839111c 100644 --- a/src/indexer/build-cache/location.ts +++ b/src/indexer/build-cache/location.ts @@ -55,9 +55,10 @@ function resolveCodegraphUserCacheRoot(): string { return path.join(base, "codegraph"); } -export function projectCacheNamespace(projectRoot: string): string { - const rootIdentity = fileIdentityKey(path.resolve(projectRoot)); - const hash = crypto.createHash("sha256").update(rootIdentity).digest("hex"); +export function projectCacheNamespace(projectRoot: string, anchor?: string): string { + const root = path.resolve(projectRoot); + const identity = anchor ? fileIdentityKey(path.relative(path.resolve(anchor), root)) || "." : fileIdentityKey(root); + const hash = crypto.createHash("sha256").update(identity).digest("hex"); return `project-${hash}`; } @@ -96,7 +97,10 @@ export function resolveCacheLocation(projectRoot: string, opts?: BuildOptions): ) { return { path: path.join(root, ".codegraph-cache", "index-v1"), anchor, layer: effectiveLayer }; } - const namespace = projectCacheNamespace(root); + const namespace = projectCacheNamespace( + root, + effectiveLayer === "git" || effectiveLayer === "manifest" ? anchor : undefined, + ); const explicitBase = opts?.cacheDir?.trim() || process.env.CODEGRAPH_CACHE_DIR?.trim(); if (explicitBase) { const configured = path.resolve(explicitBase); diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index fdd90097..20d2b442 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -235,7 +235,7 @@ function transformSnapshotPaths( copy.bloomFilters = bloomFilters; } const fileSignatures: Record = {}; - for (const [file, signature] of Object.entries(copy.fileSignatures)) { + for (const [file, signature] of Object.entries(copy.fileSignatures ?? {})) { fileSignatures[transformPath(root, file, toRelative)] = signature; } copy.fileSignatures = fileSignatures; @@ -289,10 +289,6 @@ function migrateDetailedSymbolGraphPayload(value: unknown, currentRoot: string): graph: transformDetailedGraph(relativeGraph, currentRoot, false), }; } -function projectRootMatches(projectRoot: string, storedProjectRoot: string): boolean { - return fileIdentityKey(path.resolve(projectRoot)) === fileIdentityKey(path.resolve(storedProjectRoot)); -} - function compareSnapshotPath(left: string, right: string): number { if (left < right) return -1; if (left > right) return 1; @@ -509,7 +505,6 @@ export async function tryLoadProjectSnapshotModules( const implementationFingerprint = getImplementationFingerprint(); if ( !isProjectIndexSnapshotPayload(payload) || - !projectRootMatches(projectRoot, payload.projectRoot) || payload.nativeMode !== normalizedSnapshotNativeMode(opts?.native) || payload.nativeRuntimeFingerprint !== nativeRuntimeFingerprint || payload.implementationFingerprint !== implementationFingerprint @@ -563,7 +558,6 @@ function persistedBloomFiltersFromSidecar( if ( payload.version !== BLOOM_FILTER_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || - !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() || typeof payload.projectSnapshotIdentity !== "string" || !/^[a-f0-9]{64}$/.test(payload.projectSnapshotIdentity) @@ -592,7 +586,6 @@ function persistedBloomFiltersFromSnapshot( if ( payload.version !== PROJECT_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || - !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() ) { return null; diff --git a/src/session.ts b/src/session.ts index 2f224fb1..e234269d 100644 --- a/src/session.ts +++ b/src/session.ts @@ -80,6 +80,7 @@ function normalizeBuildOptions(options?: BuildOptions): Record return { cache: options.cache, cacheDir: options.cacheDir ? path.resolve(options.cacheDir) : undefined, + cacheLocation: options.cacheLocation, cacheStrict: options.cacheStrict, useBloomFilters: options.useBloomFilters, graph: options.graph @@ -245,7 +246,14 @@ export class CodeReviewSession implements ICodeReviewSession { const graph = mergeGraphOptions(config.graph, this.buildOptions?.graph); const languageExtensions = normalizeLanguageExtensions(this.buildOptions?.languageExtensions) ?? config.languages?.extensions; - if (!hasDiscoveryOptions(discovery) && !config.graph && !this.buildOptions?.graph && !languageExtensions) { + const cacheLocation = this.buildOptions?.cacheLocation ?? config.cache?.location; + if ( + !hasDiscoveryOptions(discovery) && + !config.graph && + !this.buildOptions?.graph && + !languageExtensions && + !cacheLocation + ) { return this.buildOptions; } return { @@ -253,6 +261,7 @@ export class CodeReviewSession implements ICodeReviewSession { ...(hasDiscoveryOptions(discovery) ? { discovery } : {}), ...(config.graph || this.buildOptions?.graph ? { graph } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 8863ccae..6cd7e5ae 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -2560,4 +2560,67 @@ describe("Cache invalidation and strict hashing", () => { expect(fileIdentityKey(resolution.anchor)).toBe(fileIdentityKey(projectRoot)); expect(resolution.layer).toBe("project"); }); + + it("keeps the repo-anchored cache namespace stable when the repository moves", async () => { + const parent = await mkTmpDir("dg-cache-namespace-move-"); + const repoRoot = path.join(parent, "repo-a"); + const projectRoot = path.join(repoRoot, "packages", "app"); + await fsp.mkdir(projectRoot, { recursive: true }); + await fsp.writeFile(path.join(repoRoot, ".git"), "gitdir: external\n", "utf8"); + const before = buildCache.cacheRoot(projectRoot, { cache: "disk" }); + + const movedRepoRoot = path.join(parent, "repo-a-renamed"); + await fsp.rename(repoRoot, movedRepoRoot); + const movedProjectRoot = path.join(movedRepoRoot, "packages", "app"); + const after = buildCache.cacheRoot(movedProjectRoot, { cache: "disk" }); + + expect(path.basename(after)).toBe(path.basename(before)); + }); + + it("reuses a legacy v4 project snapshot missing fileSignatures instead of crashing to a forced miss", async () => { + const root = await mkTmpDir("dg-cache-legacy-v4-"); + await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const manifest = await readManifest(root); + const entries = new Map(Object.entries(manifest.files)); + + const snapshotPath = projectSnapshotPathFor(root); + const snapshot = (await readProjectSnapshot(snapshotPath)) as Record; + delete snapshot.fileSignatures; + delete snapshot.nativeMode; + delete snapshot.projectFiles; + delete snapshot.bloomFilters; + snapshot.version = 4; + await writeProjectSnapshot(snapshotPath, snapshot); + + // Previously, iterating the missing `fileSignatures` field during v4 migration threw and + // was swallowed by the outer try/catch, forcing a cache miss even when the rest of the + // migrated payload (fingerprints, files signature, graph) was otherwise still compatible. + const loaded = await buildCache.tryLoadProjectIndexSnapshot(root, { cache: "disk" }, entries); + expect(loaded).not.toBeNull(); + expect(loaded?.index.byFile.has(fileIdentityKey(normalize(path.join(root, "entry.ts"))))).toBe(true); + + const rebuilt = await buildProjectIndexIncremental(root, { threads: 2, cache: "disk" }); + expect(rebuilt.byFile.size).toBeGreaterThan(0); + }); + + it("reuses per-file modules and bloom filters after a project move even when a sibling file changed", async () => { + const sourceRoot = await mkTmpDir("dg-cache-partial-move-source-"); + const movedRoot = `${sourceRoot}-moved`; + await fsp.writeFile(path.join(sourceRoot, "unchanged.ts"), "export const unchanged = 1;\n", "utf8"); + await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export const entry = 1;\n", "utf8"); + await buildProjectIndex(sourceRoot, { cache: "disk", threads: 1 }); + await fsp.rename(sourceRoot, movedRoot); + + const bloomFilters = await buildCache.tryLoadPersistedBloomFilters(movedRoot, { cache: "disk" }); + expect(bloomFilters).not.toBeNull(); + + await fsp.writeFile(path.join(movedRoot, "entry.ts"), "export const entry = 2;\n", "utf8"); + const report: BuildReport = { timings: {} }; + const rebuilt = await buildProjectIndexIncremental(movedRoot, { cache: "disk", threads: 1, report }); + + expect(rebuilt.byFile.has(fileIdentityKey(normalize(path.join(movedRoot, "unchanged.ts"))))).toBe(true); + expect(report.files?.cached).toBeGreaterThan(0); + expect(report.cache?.misses ?? 0).toBeLessThanOrEqual(1); + }); }); diff --git a/tests/cli-command-modules.test.ts b/tests/cli-command-modules.test.ts index 52ce4d1b..9355ee84 100644 --- a/tests/cli-command-modules.test.ts +++ b/tests/cli-command-modules.test.ts @@ -1237,6 +1237,10 @@ describe("CLI command modules", () => { expect(pretty).toMatchObject({ stderr: "", exitCode: undefined }); expect(pretty.stdout).toContain("Package:\n Name: @lzehrung/codegraph"); expect(pretty.stdout).toMatch(/^Package:/m); + expect(pretty.stdout).toMatch(/^Cache:/m); + expect(pretty.stdout).toMatch(/^ {2}Path: /m); + expect(pretty.stdout).toMatch(/^ {2}Anchor: /m); + expect(pretty.stdout).toMatch(/^ {2}Layer: /m); expect(pretty.stdout).toMatch(/^Native:/m); expect(pretty.stdout).toMatch(/^ {2}Origin:/m); expect(pretty.stdout).toMatch(/^ {2}Update:/m); diff --git a/tests/cli-regressions.test.ts b/tests/cli-regressions.test.ts index ff2f36b5..3ad6352b 100644 --- a/tests/cli-regressions.test.ts +++ b/tests/cli-regressions.test.ts @@ -11,6 +11,7 @@ import { runGit as git } from "./helpers/git.js"; import { captureCli, runCliOrThrow, runTsxScriptOrThrow } from "./helpers/cli.js"; import { copyFixtureSubset, createTwoCommitCycleProject, readOnlySamplePath } from "./helpers/filesystem.js"; import { decompactFileGraph, type CompactFileGraphPayload } from "./helpers/compactGraph.js"; +import { cacheRoot } from "../src/indexer/build-cache/location.js"; const sourceCliPath = path.resolve(process.cwd(), "src", "cli.ts"); @@ -999,6 +1000,35 @@ describe("CLI regressions", () => { expect(result.stderr).toContain("lastCommit="); }); + it("honors --cache-dir for index and goto instead of silently writing to the default location", async () => { + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-cli-cache-dir-")); + await fsp.writeFile(path.join(tmpDir, "main.ts"), "export const main = 1;\n", "utf8"); + await fsp.writeFile( + path.join(tmpDir, "usage.ts"), + "import { main } from './main';\nexport const used = main;\n", + "utf8", + ); + const cacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-cli-cache-dir-target-")); + const expectedCachePath = cacheRoot(tmpDir, { cache: "disk", cacheDir }); + + await runCliCommand(["index", "--json", "--root", tmpDir, "--cache-dir", cacheDir]); + await expect(fsp.stat(path.join(expectedCachePath, "manifest.json"))).resolves.toBeTruthy(); + await expect(fsp.stat(path.join(tmpDir, ".codegraph-cache", "index-v1", "manifest.json"))).rejects.toThrow(); + + await fsp.rm(expectedCachePath, { recursive: true, force: true }); + const gotoStdout = await runCliCommand([ + "goto", + `${path.join(tmpDir, "usage.ts")}:1:10`, + "--root", + tmpDir, + "--cache-dir", + cacheDir, + "--json", + ]); + expect(JSON.parse(gotoStdout)).toMatchObject({ status: "ok" }); + await expect(fsp.stat(path.join(expectedCachePath, "manifest.json"))).resolves.toBeTruthy(); + }); + it("inspect emits backend, file summary, scoped hotspots, and recommended commands", async () => { const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-cli-inspect-")); const srcDir = path.join(tmpDir, "src"); diff --git a/tests/query-index.test.ts b/tests/query-index.test.ts index f5160139..27d2b3a4 100644 --- a/tests/query-index.test.ts +++ b/tests/query-index.test.ts @@ -761,6 +761,30 @@ describe("persistent query index", () => { store.close(); }); + it("bounds candidateChunksForTerms SQL prefetch via the limit parameter", async () => { + const root = await createRepo(); + const databasePath = path.join(root, "query-candidate-limit.sqlite"); + const store = new QueryIndexStore(databasePath); + const files = Array.from({ length: 20 }, (_, index) => + preparedFile(`src/file${String(index).padStart(2, "0")}.ts`, ["const alphaNoise = 1;"]), + ); + store.replaceFiles(files, [], { + ...expectedQueryIndexVersionMetadata(), + projectSnapshotIdentity: "snap-4", + projectRootIdentity: "root-4", + createdByCodegraphVersion: "test", + updatedAt: new Date().toISOString(), + }); + const paths = files.map((file) => file.path); + + const unbounded = store.candidateChunksForTerms(["alpha"], paths); + expect(unbounded).toHaveLength(20); + + const bounded = store.candidateChunksForTerms(["alpha"], paths, 5); + expect(bounded).toHaveLength(5); + store.close(); + }); + it("rejects absolute and traversing paths from persisted rows", async () => { const root = await createRepo(); expect(() => resolveQueryIndexSourcePath(root, "../outside.ts")).toThrow(/Invalid query index relative path/u); diff --git a/tests/session.test.ts b/tests/session.test.ts index df171253..510cf11d 100644 --- a/tests/session.test.ts +++ b/tests/session.test.ts @@ -827,6 +827,33 @@ index 1234567..abcdef0 100644 } }); + test("should merge codegraph.config.json cache.location into session build options", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-session-cache-location-")); + try { + await fsp.writeFile(path.join(root, "main.ts"), "export const value = 1;\n", "utf8"); + const cacheLocation = path.join(root, "custom-cache"); + await fsp.mkdir(cacheLocation, { recursive: true }); + await fsp.writeFile( + path.join(root, "codegraph.config.json"), + JSON.stringify({ cache: { location: cacheLocation } }), + "utf8", + ); + + const buildSpy = vi.spyOn(indexerBuild, "buildProjectIndexIncremental"); + try { + const session = await createCodeReviewSession({ root, buildOptions: { cache: "memory" } }); + expect(session.getStatus()).toBe("ready"); + expect(buildSpy.mock.calls.length).toBeGreaterThan(0); + const options = buildSpy.mock.calls[0]?.[1] as BuildOptions | undefined; + expect(options?.cacheLocation).toBe(cacheLocation); + } finally { + buildSpy.mockRestore(); + } + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); + test("should include new source files on manual refresh", async () => { const root = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-session-manual-added-file-")); try { @@ -1397,6 +1424,20 @@ describe("SessionManager", () => { ).rejects.toThrow(/different configuration/); }); + test("should reject reusing a session id when cacheLocation drifts", async () => { + await manager.getOrCreateSession("shared", { + root: sampleRoot, + buildOptions: sampleBuildOptions({ cacheLocation: "project" }), + }); + + await expect( + manager.getOrCreateSession("shared", { + root: sampleRoot, + buildOptions: sampleBuildOptions({ cacheLocation: "user" }), + }), + ).rejects.toThrow(/different configuration/); + }); + test("should reuse a session when languageExtensions is empty instead of omitted", async () => { const session1 = await manager.getOrCreateSession("shared", { root: sampleRoot, From ea40ee857527bdbf7527dda83cd13ced1bff2dbe Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 18:21:23 -0400 Subject: [PATCH 23/28] fix: address second batch of suppressed PR review findings - finalize.ts/build-index.ts: resolve normalizedProjectRoot to an absolute path and use it for ProjectIndex.projectRoot on cold/warm incremental builds, matching the always-absolute snapshot-hydrated path. Previously a caller passing a relative root (e.g. ".") got a relative projectRoot on cold builds, making later duplicate-cache path math depend on process.cwd() at query time. - duplicates/unitCache.ts + units.ts: thread a projectRoot override through writeDuplicateUnitsBatchToCache (added in this PR's write batching), matching the read path and the older single-write function. Reads already respected a projectRoot override different from index.projectRoot; the batched writer silently ignored it, so scoped duplicate analysis could never hit its own disk cache. - location.ts: treat "user" and "environment" cache anchors as creatable, like "explicit", instead of falling back to layer "project" when the target directory doesn't exist yet. The actual cache path was already correct (base/candidate computation ignores anchor for those layers); only the reported anchor/layer metadata was wrong, contradicting doctor diagnostics and the resolved path. - doctor.ts: read cache.location from a project's codegraph.config.json so the cache diagnostic matches what index/search/etc. would actually use. Uses a small dependency-free JSON peek (not the zod-based ../config.js) to stay inside the enforced <30-dist-module startup budget for the doctor command (loading config.js pushed it to 71). - inspect.ts: thread --cache-dir and configured cacheLocation through InspectCommandContext, the shared index loaders, and the cache-metadata helpers backing inspect/hotspots, so they stop silently reading the default cache location and reporting the wrong manifest path. Adds regression coverage in finalize-project-index.test.ts, cache-invalidation.test.ts, disk-cache-sqlite.test.ts, cli-command-modules.test.ts, and cli-regressions.test.ts. --- src/cli/commandTable.ts | 2 ++ src/cli/doctor.ts | 20 ++++++++++- src/cli/inspect.ts | 54 ++++++++++++++++++++++------ src/duplicates/unitCache.ts | 7 ++-- src/duplicates/units.ts | 2 +- src/indexer/build-cache/location.ts | 4 ++- src/indexer/build-index.ts | 2 +- src/indexer/finalize.ts | 2 +- tests/cache-invalidation.test.ts | 26 ++++++++++++++ tests/cli-command-modules.test.ts | 21 +++++++++++ tests/cli-regressions.test.ts | 11 +++++- tests/disk-cache-sqlite.test.ts | 39 +++++++++++++++++++- tests/finalize-project-index.test.ts | 17 +++++++++ 13 files changed, 187 insertions(+), 20 deletions(-) diff --git a/src/cli/commandTable.ts b/src/cli/commandTable.ts index 6188b6e7..5e067cce 100644 --- a/src/cli/commandTable.ts +++ b/src/cli/commandTable.ts @@ -662,6 +662,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { includeRootsAbs: ctx.includeRootsAbs, discoveryOptions: ctx.discoveryOptions, languageExtensions: ctx.config.languages?.extensions, + cacheLocation: ctx.config.cache?.location, graphOptions: ctx.hasGraphOverrides || ctx.nativeMode !== "auto" ? ctx.buildGraphOptions() : undefined, nativeMode: ctx.nativeMode, workerOpts: ctx.workerOpts, @@ -687,6 +688,7 @@ export const CLI_COMMAND_TABLE: Readonly> = { includeRootsAbs: ctx.includeRootsAbs, discoveryOptions: ctx.discoveryOptions, languageExtensions: ctx.config.languages?.extensions, + cacheLocation: ctx.config.cache?.location, graphOptions: ctx.hasGraphOverrides || ctx.nativeMode !== "auto" ? ctx.buildGraphOptions() : undefined, nativeMode: ctx.nativeMode, workerOpts: ctx.workerOpts, diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 59517bb5..e663a1c5 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -319,6 +319,23 @@ export function findStaleNpmRetirementPaths(packageRoot: string, limit = 20): st return []; } } +/** + * Best-effort, dependency-light read of `cache.location` from `codegraph.config.json` in the + * current working directory. Deliberately avoids importing `../config.js` (which pulls in zod + * and the full config schema): doctor is a fast, low-dependency health check, and the eager + * dist-module-loading budget in `cli-startup-eager-modules.test.ts` enforces that. Malformed or + * missing config is silently ignored; the fuller schema validation still applies to real builds. + */ +function readProjectCacheLocation(root: string): string | undefined { + try { + const raw = fs.readFileSync(path.join(root, "codegraph.config.json"), "utf8"); + const parsed = JSON.parse(raw) as { cache?: { location?: unknown } }; + const location = parsed.cache?.location; + return typeof location === "string" && location.trim() ? location.trim() : undefined; + } catch { + return undefined; + } +} export function buildDoctorReport(indexPath?: string): DoctorReport { const packageIdentity = getCodegraphPackageIdentity(); @@ -326,7 +343,8 @@ export function buildDoctorReport(indexPath?: string): DoctorReport { const origin = getNativeBindingOrigin(); const runtimeIdentity = captureCodegraphRuntimeIdentity(origin); const update = createInstalledVersionChecker(runtimeIdentity, { warn: () => undefined }).check(true); - const cacheResolution = resolveCacheLocation(process.cwd()); + const cacheLocation = readProjectCacheLocation(process.cwd()); + const cacheResolution = resolveCacheLocation(process.cwd(), cacheLocation ? { cacheLocation } : undefined); return { package: packageIdentity, cache: { diff --git a/src/cli/inspect.ts b/src/cli/inspect.ts index a96c616b..a22257c4 100644 --- a/src/cli/inspect.ts +++ b/src/cli/inspect.ts @@ -7,7 +7,7 @@ import { findDetailedCycles, getUnresolvedImports } from "../graphs/queries.js"; import { getHotspots } from "../graphs/hotspots.js"; import type { GraphBuildOptions } from "../graphs/types.js"; import { loadCurrentProjectIndex } from "../indexer/load-current-index.js"; -import { type BuildOptions, type BuildReport } from "../indexer/types.js"; +import { type BuildOptions, type BuildReport, type CacheLocation } from "../indexer/types.js"; import { getNativeTreeSitterLoadError, getNativeTreeSitterSupportedLanguageIds, @@ -107,6 +107,7 @@ export type InspectCommandContext = { nativeMode: NativeRuntimeMode; workerOpts: { useNativeWorkers: true } | Record; progressHandler: BuildOptions["onProgress"]; + cacheLocation: CacheLocation | undefined; getOpt: (name: string) => string | undefined; hasFlag: (name: string) => boolean; resolveFilesFromRoots: () => Promise; @@ -118,16 +119,32 @@ export type InspectCommandContext = { writeCommandReport?: (report: CommandReport, reportFile: string | undefined) => Promise; }; -function defaultCacheIndexPath(projectRoot: string): string { - return cacheRoot(projectRoot, { cache: "disk" }); +function defaultCacheIndexPath( + projectRoot: string, + cacheDir: string | undefined, + cacheLocation: CacheLocation | undefined, +): string { + return cacheRoot(projectRoot, { + cache: "disk", + ...(cacheDir ? { cacheDir } : {}), + ...(cacheLocation ? { cacheLocation } : {}), + }); } -function defaultCacheManifestPath(projectRoot: string): string { - return path.join(defaultCacheIndexPath(projectRoot), "manifest.json"); +function defaultCacheManifestPath( + projectRoot: string, + cacheDir: string | undefined, + cacheLocation: CacheLocation | undefined, +): string { + return path.join(defaultCacheIndexPath(projectRoot, cacheDir, cacheLocation), "manifest.json"); } -function readIndexCacheMetadata(projectRoot: string): IndexCacheMetadata | null { - const manifestPath = defaultCacheManifestPath(projectRoot); +function readIndexCacheMetadata( + projectRoot: string, + cacheDir: string | undefined, + cacheLocation: CacheLocation | undefined, +): IndexCacheMetadata | null { + const manifestPath = defaultCacheManifestPath(projectRoot, cacheDir, cacheLocation); try { const raw = fs.readFileSync(manifestPath, "utf8"); const parsed = JSON.parse(raw) as { @@ -156,6 +173,8 @@ async function buildScopedReportGraph( files: string[], opts: { cache?: CacheMode; + cacheDir?: string; + cacheLocation?: CacheLocation; discovery?: ProjectFileDiscoveryOptions; languageExtensions?: LanguageExtensionMap; graphOptions?: GraphBuildOptions; @@ -169,7 +188,7 @@ async function buildScopedReportGraph( // Cache metadata is reporting only: the shared loader owns freshness, so a cold run // builds reusable state here instead of collecting a throwaway graph. const useDiskCache = opts.cache === "disk" || opts.cache === undefined; - const indexCache = useDiskCache ? readIndexCacheMetadata(projectRoot) : null; + const indexCache = useDiskCache ? readIndexCacheMetadata(projectRoot, opts.cacheDir, opts.cacheLocation) : null; if (indexCache) { opts.writeStderrLine(formatIndexCacheMetadata(indexCache)); } @@ -178,6 +197,8 @@ async function buildScopedReportGraph( scope: { kind: "resolved-files", files }, options: { ...(opts.cache ? { cache: opts.cache } : {}), + ...(opts.cacheDir ? { cacheDir: opts.cacheDir } : {}), + ...(opts.cacheLocation ? { cacheLocation: opts.cacheLocation } : {}), ...(opts.discovery ? { discovery: opts.discovery } : {}), ...(opts.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts.progressHandler ? { onProgress: opts.progressHandler } : {}), @@ -232,6 +253,8 @@ function buildRecommendedInspectCommands( includeRoots: string[], hasCycles: boolean, hasUnresolvedImports: boolean, + cacheDir: string | undefined, + cacheLocation: CacheLocation | undefined, ): string[] { const rootFlag = `--root "${normalizePath(projectRoot)}"`; const targetSuffix = includeRoots.length @@ -248,7 +271,7 @@ function buildRecommendedInspectCommands( if (hasCycles) { commands.push(`codegraph cycles ${rootFlag}${targetSuffix} --sort priority --json`); } - commands.push(`codegraph doctor "${normalizePath(defaultCacheIndexPath(projectRoot))}"`); + commands.push(`codegraph doctor "${normalizePath(defaultCacheIndexPath(projectRoot, cacheDir, cacheLocation))}"`); return commands; } function formatInspectLanguageCounts(byLanguage: Record): string { @@ -349,6 +372,8 @@ async function buildInspectReport( graphOptions: GraphBuildOptions | undefined, languageExtensions: LanguageExtensionMap | undefined, cache: CacheMode | undefined, + cacheDir: string | undefined, + cacheLocation: CacheLocation | undefined, nativeMode: NativeRuntimeMode, workerOpts: { useNativeWorkers: true } | Record, progressHandler: BuildOptions["onProgress"], @@ -358,7 +383,7 @@ async function buildInspectReport( writeStderrLine: (message: string) => void, ): Promise { const useDiskCache = cache === "disk" || cache === undefined; - const indexCache = useDiskCache ? readIndexCacheMetadata(projectRoot) : null; + const indexCache = useDiskCache ? readIndexCacheMetadata(projectRoot, cacheDir, cacheLocation) : null; if (indexCache) { writeStderrLine(formatIndexCacheMetadata(indexCache)); } @@ -368,6 +393,8 @@ async function buildInspectReport( scope: { kind: "resolved-files", files }, options: { ...(cache ? { cache } : {}), + ...(cacheDir ? { cacheDir } : {}), + ...(cacheLocation ? { cacheLocation } : {}), discovery, ...(languageExtensions ? { languageExtensions } : {}), ...(progressHandler ? { onProgress: progressHandler } : {}), @@ -442,6 +469,8 @@ async function buildInspectReport( includeRoots, !!cycles.length, !!unresolved.length, + cacheDir, + cacheLocation, ), }; } @@ -463,6 +492,8 @@ export async function handleInspectCommand(context: InspectCommandContext): Prom context.graphOptions, context.languageExtensions, cache, + context.getOpt("--cache-dir"), + context.cacheLocation, context.nativeMode, context.workerOpts, context.progressHandler, @@ -484,8 +515,11 @@ export async function handleHotspotsCommand(context: InspectCommandContext): Pro const cache = parseCacheModeOption(context.getOpt("--cache")); const limit = parsePositiveIntegerOption(context.getOpt("--limit"), "--limit", 20); const files = await context.resolveFilesFromRoots(); + const cacheDir = context.getOpt("--cache-dir"); const { graph } = await buildScopedReportGraph(context.projectRootFs, context.includeRootsAbs, files, { ...(cache ? { cache } : {}), + ...(cacheDir ? { cacheDir } : {}), + ...(context.cacheLocation ? { cacheLocation: context.cacheLocation } : {}), discovery: context.discoveryOptions, ...(context.languageExtensions ? { languageExtensions: context.languageExtensions } : {}), ...(context.graphOptions ? { graphOptions: context.graphOptions } : {}), diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 51c62d59..8ff82dbe 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -378,12 +378,13 @@ export type PendingDuplicateUnitCacheWrite = { export function writeDuplicateUnitsBatchToCache( index: ProjectIndex, writes: readonly PendingDuplicateUnitCacheWrite[], + projectRoot?: string, ): void { if (!writes.length) return; - const root = index.projectRoot ?? ""; + const root = projectRoot ?? index.projectRoot ?? ""; if (index.cacheMode === "memory") { for (const write of writes) { - const sig = duplicateUnitCacheSignature(index, write.file); + const sig = duplicateUnitCacheSignature(index, write.file, projectRoot); if (!sig) continue; writeDuplicateUnitMemoryCache(duplicateUnitCacheKey(write.file, write.variant), { sig, @@ -402,7 +403,7 @@ export function writeDuplicateUnitsBatchToCache( payload: Buffer; }> = []; for (const write of writes) { - const sig = duplicateUnitCacheSignature(index, write.file); + const sig = duplicateUnitCacheSignature(index, write.file, projectRoot); if (!sig) continue; const payload = brotliCompressSync( JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(write.units), true)), diff --git a/src/duplicates/units.ts b/src/duplicates/units.ts index 25c501fb..7f70ef3a 100644 --- a/src/duplicates/units.ts +++ b/src/duplicates/units.ts @@ -564,7 +564,7 @@ export async function collectDuplicateUnits( } } if (pendingWrites.length) { - writeDuplicateUnitsBatchToCache(index, pendingWrites); + writeDuplicateUnitsBatchToCache(index, pendingWrites, options.projectRoot); } units.sort((left, right) => { diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts index a839111c..60e1b634 100644 --- a/src/indexer/build-cache/location.ts +++ b/src/indexer/build-cache/location.ts @@ -85,7 +85,9 @@ export type CacheLocationResolution = CacheAnchorResolution & { path: string }; export function resolveCacheLocation(projectRoot: string, opts?: BuildOptions): CacheLocationResolution { const root = path.resolve(projectRoot); const resolution = resolveCacheAnchor(root, opts); - const anchorWritable = resolution.layer === "explicit" || isWritableDirectory(resolution.anchor); + const anchorCreatable = + resolution.layer === "explicit" || resolution.layer === "environment" || resolution.layer === "user"; + const anchorWritable = anchorCreatable || isWritableDirectory(resolution.anchor); const anchor = anchorWritable ? resolution.anchor : root; const effectiveLayer = anchorWritable ? resolution.layer : "project"; const sameRoot = fileIdentityKey(anchor) === fileIdentityKey(root); diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index d0e3f315..61b83c6b 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -523,7 +523,7 @@ function createIndexBuildRunState( if (report) initNativeBackendReport(report); const cacheMode = opts?.cache ?? "off"; return { - normalizedProjectRoot: normalizePath(projectRoot), + normalizedProjectRoot: normalizePath(path.resolve(projectRoot)), report, timings: report?.timings, totalStart: performance.now(), diff --git a/src/indexer/finalize.ts b/src/indexer/finalize.ts index f39fcf36..2e6cb2e4 100644 --- a/src/indexer/finalize.ts +++ b/src/indexer/finalize.ts @@ -32,7 +32,7 @@ export async function finalizeProjectIndex(args: { const languageExtensions = normalizeLanguageExtensions(args.opts?.languageExtensions); const parsed = retainedParsedCache(args.parsedMap, args.opts); return { - projectRoot: args.projectRoot, + projectRoot: args.normalizedProjectRoot, graph: args.graph, graphAdjacency: buildGraphAdjacency(args.graph), modules: args.modules, diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 6cd7e5ae..00bb1852 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -2577,6 +2577,22 @@ describe("Cache invalidation and strict hashing", () => { expect(path.basename(after)).toBe(path.basename(before)); }); + it("reports the environment cache anchor even when CODEGRAPH_CACHE_DIR does not exist yet", async () => { + const root = await mkTmpDir("dg-cache-env-anchor-"); + await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); + const envParent = await mkTmpDir("dg-cache-env-target-"); + const envTarget = path.join(envParent, "not-created-yet"); + vi.stubEnv("CODEGRAPH_CACHE_DIR", envTarget); + try { + const resolution = buildCache.resolveCacheLocation(root, { cache: "disk" }); + expect(fileIdentityKey(resolution.anchor)).toBe(fileIdentityKey(path.resolve(envTarget))); + expect(resolution.layer).toBe("environment"); + expect(normalize(resolution.path).startsWith(normalize(path.resolve(envTarget)))).toBe(true); + } finally { + vi.unstubAllEnvs(); + } + }); + it("reuses a legacy v4 project snapshot missing fileSignatures instead of crashing to a forced miss", async () => { const root = await mkTmpDir("dg-cache-legacy-v4-"); await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); @@ -2623,4 +2639,14 @@ describe("Cache invalidation and strict hashing", () => { expect(report.files?.cached).toBeGreaterThan(0); expect(report.cache?.misses ?? 0).toBeLessThanOrEqual(1); }); + + it("resolves ProjectIndex.projectRoot to an absolute path even when a relative root is passed in", async () => { + const root = await mkTmpDir("dg-cache-relative-root-"); + await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); + const relativeRoot = path.relative(process.cwd(), root); + + const index = await buildProjectIndex(relativeRoot, { cache: "off", threads: 1 }); + + expect(index.projectRoot).toBe(normalize(path.resolve(root))); + }); }); diff --git a/tests/cli-command-modules.test.ts b/tests/cli-command-modules.test.ts index 9355ee84..67ab6e54 100644 --- a/tests/cli-command-modules.test.ts +++ b/tests/cli-command-modules.test.ts @@ -1384,6 +1384,27 @@ describe("CLI command modules", () => { } }); + test("doctor reports the effective cache.location from codegraph.config.json", async () => { + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-cache-config-")); + const cacheLocation = path.join(tempDir, "custom-cache"); + await fsp.mkdir(cacheLocation, { recursive: true }); + await fsp.writeFile( + path.join(tempDir, "codegraph.config.json"), + JSON.stringify({ cache: { location: cacheLocation } }), + "utf8", + ); + const previousCwd = process.cwd(); + process.chdir(tempDir); + try { + const report = buildDoctorReport(); + expect(report.cache.layer).toBe("explicit"); + expect(report.cache.anchor).toBe(cacheLocation.replace(/\\/g, "/")); + } finally { + process.chdir(previousCwd); + await fsp.rm(tempDir, { recursive: true, force: true }); + } + }); + test("builds doctor reports for explicit index artifact paths", async () => { const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-module-")); const artifactPath = path.join(tempDir, "codegraph.json"); diff --git a/tests/cli-regressions.test.ts b/tests/cli-regressions.test.ts index 3ad6352b..70369669 100644 --- a/tests/cli-regressions.test.ts +++ b/tests/cli-regressions.test.ts @@ -1000,7 +1000,7 @@ describe("CLI regressions", () => { expect(result.stderr).toContain("lastCommit="); }); - it("honors --cache-dir for index and goto instead of silently writing to the default location", async () => { + it("honors --cache-dir for index, goto, hotspots, and inspect instead of silently writing to the default location", async () => { const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "dg-cli-cache-dir-")); await fsp.writeFile(path.join(tmpDir, "main.ts"), "export const main = 1;\n", "utf8"); await fsp.writeFile( @@ -1027,6 +1027,15 @@ describe("CLI regressions", () => { ]); expect(JSON.parse(gotoStdout)).toMatchObject({ status: "ok" }); await expect(fsp.stat(path.join(expectedCachePath, "manifest.json"))).resolves.toBeTruthy(); + + await fsp.rm(expectedCachePath, { recursive: true, force: true }); + const hotspotsStdout = await runCliCommand(["hotspots", "--root", tmpDir, "--cache-dir", cacheDir, "--json"]); + expect(JSON.parse(hotspotsStdout)).toBeInstanceOf(Array); + await expect(fsp.stat(path.join(expectedCachePath, "manifest.json"))).resolves.toBeTruthy(); + + const inspectStdout = await runCliCommand(["inspect", "--root", tmpDir, "--cache-dir", cacheDir, "--json"]); + const inspectReport = JSON.parse(inspectStdout) as { indexCache?: { manifestPath?: string } }; + expect(normalize(inspectReport.indexCache?.manifestPath ?? "")).toContain(normalize(cacheDir)); }); it("inspect emits backend, file summary, scoped hotspots, and recommended commands", async () => { diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 27f028dc..c30c1b18 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -6,7 +6,11 @@ import { brotliCompressSync, brotliDecompressSync } from "node:zlib"; import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/index.js"; import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; -import { tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "../src/duplicates/unitCache.js"; +import { + tryLoadDuplicateUnitsFromCache, + writeDuplicateUnitsBatchToCache, + writeDuplicateUnitsToCache, +} from "../src/duplicates/unitCache.js"; import { buildInternalUnit, formatDuplicateSqlHandle, formatDuplicateSymbolHandle } from "../src/duplicates/units.js"; import * as buildCache from "../src/indexer/build-cache.js"; import { SqliteDatabase, SqliteStatement } from "../src/sqlite-driver.js"; @@ -564,6 +568,39 @@ describe("disk cache uses sqlite backend", () => { expect(loaded?.[1]?.symbolHandle).toBe(persistedEmptyNameHandle); }); + it("honors a projectRoot override in the batched duplicate-unit cache writer", async () => { + const root = await mkTmpDir("dg-disk-cache-batch-scoped-root-"); + await writeDuplicateProject(root); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const file = normalizePathForSql(path.join(root, "src", "a.ts")); + const source = await fsp.readFile(file, "utf8"); + const scopedRoot = path.join(root, "src"); + const unit = buildInternalUnit( + { + file: "src/a.ts", + startLine: 1, + endLine: 8, + languageId: "typescript", + kind: "symbol", + name: "normalizeInvoiceRows", + }, + file, + source, + 3, + 2, + index.nativeMode, + { symbolHandle: formatDuplicateSymbolHandle(file, "normalizeInvoiceRows", 1, 0) }, + ); + + writeDuplicateUnitsBatchToCache(index, [{ file, variant: "batch-scoped", units: [unit] }], scopedRoot); + + const loadedWithScopedRoot = tryLoadDuplicateUnitsFromCache(index, file, "batch-scoped", scopedRoot); + expect(loadedWithScopedRoot?.[0]?.id).toBe(unit.id); + + const loadedWithIndexRoot = tryLoadDuplicateUnitsFromCache(index, file, "batch-scoped", index.projectRoot); + expect(loadedWithIndexRoot).toBeNull(); + }); + it("rejects duplicate units whose persisted absolute file escapes the project", async () => { const root = await mkTmpDir("dg-disk-cache-duplicate-unit-confinement-"); await writeDuplicateProject(root); diff --git a/tests/finalize-project-index.test.ts b/tests/finalize-project-index.test.ts index c798768a..770aef12 100644 --- a/tests/finalize-project-index.test.ts +++ b/tests/finalize-project-index.test.ts @@ -44,4 +44,21 @@ describe("finalizeProjectIndex", () => { expect(index.projectFiles).toEqual(projectFiles); expect(mocks.discoverProjectFiles).not.toHaveBeenCalled(); }); + + it("returns the resolved normalizedProjectRoot instead of the raw possibly-relative projectRoot", async () => { + const index = await finalizeProjectIndex({ + projectRoot: ".", + normalizedProjectRoot: "/repo", + opts: undefined, + timings: undefined, + totalStart: performance.now(), + graph: { nodes: new Set(), edges: [] }, + modules: new Map(), + parsedMap: new Map(), + bloomFilterCache: undefined, + projectFiles: Promise.resolve([]), + }); + + expect(index.projectRoot).toBe("/repo"); + }); }); From d2c50af016cc0623a293181fcda26f4b2d181465 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 23:25:47 -0400 Subject: [PATCH 24/28] fix: address third batch of review findings - manifest, session config, and languageExtensions threading High-priority: - build-cache/manifest.ts: rebase legacy v3 transientFiles (persisted as absolute paths) from the manifest's stored projectRoot before sanitizing against the active root, matching the symlinkDirectories migration pattern. Previously every transient/additional file was silently dropped after moving a legacy project tree. Suppressed findings: - agent/session.ts: merge codegraph.config.json's cache.location into createAgentSession()'s resolved build options (discoveryOptions/ graphOptions/languageExtensions already did this); explicit buildOptions.cacheLocation still wins. Threads through the incremental build, the detailed-symbol-graph cache options, and the incremental file-plan lookup. - cli/doctor.ts: read cache.location with project-over-user precedence (matching loadCodegraphConfig) via a small dependency-free JSON peek at both codegraph.config.json and the platform user config, instead of only the project file. Verified this keeps codegraph doctor under the enforced <30-dist-module startup budget (importing ../config.js pushed it to 71). - indexer/parse-context.ts (ensureParsedContext) consumers: thread index.languageExtensions through every index-backed call that was missing it (agent/renamePreview.ts, impact/{map,callCompatibility, referenceCache,suggestions}.ts, indexer/{navigation-goto, navigation-references,navigation,workspace-symbols}.ts). Previously these silently fell back to guessing a file's language from its raw extension, which throws for configured custom extensions - caught by each call site's try/catch and treated as a skipped/omitted file. - indexer/types.ts + docs/library-api.md: document BuildOptions.cacheDir/ cacheLocation precedence and anchor-vs-final-path semantics, and add a "Cache location" library-api.md section covering session config precedence. Adds regression coverage in cache-invalidation.test.ts (v3 transientFiles move), agent-session.test.ts (config cache.location merge + explicit override precedence), cli-command-modules.test.ts (doctor project- and user-config cache.location), and workspace-symbols.test.ts (import binding resolution in a custom-extension file, verified to fail without the fix). --- docs/library-api.md | 19 ++++++++++++ src/agent/renamePreview.ts | 13 +++++++-- src/agent/session.ts | 16 +++++++++-- src/cli/doctor.ts | 25 ++++++++++++---- src/impact/callCompatibility.ts | 22 ++++++++------ src/impact/map.ts | 4 +-- src/impact/referenceCache.ts | 2 +- src/impact/suggestions.ts | 6 +++- src/indexer/build-cache/manifest.ts | 13 +++++++-- src/indexer/build-index.ts | 8 ++++-- src/indexer/navigation-goto.ts | 4 +-- src/indexer/navigation-references.ts | 3 +- src/indexer/navigation.ts | 13 +++++++-- src/indexer/types.ts | 8 ++++++ src/indexer/workspace-symbols.ts | 6 +++- tests/agent-session.test.ts | 31 ++++++++++++++++++++ tests/cache-invalidation.test.ts | 43 ++++++++++++++++++++++++++++ tests/cli-command-modules.test.ts | 26 +++++++++++++++++ tests/workspace-symbols.test.ts | 28 ++++++++++++++++++ 19 files changed, 254 insertions(+), 36 deletions(-) diff --git a/docs/library-api.md b/docs/library-api.md index e741779b..43fbf5ab 100644 --- a/docs/library-api.md +++ b/docs/library-api.md @@ -624,6 +624,25 @@ const incremental = await buildProjectIndexIncremental(root, { `BuildOptions.onProgress` reports index lifecycle and file progress. A rebuild emits `phase: "start"` with `mode: "build"` or `"update"`, zero or more `phase: "update"` events, and `phase: "complete"` with `elapsedMs`; a reusable snapshot emits no progress events. +## Cache location + +`BuildOptions.cacheDir` and `BuildOptions.cacheLocation` control where the disk cache +(`cache: "disk"`) is written; they have no effect for `cache: "memory"` or `"off"`. Persisted +cache contents store project-relative paths, so a cache can be moved along with its project. + +Anchor selection precedence: `cacheDir` (an explicit final directory), then `CODEGRAPH_CACHE_DIR`, +then `cacheLocation`, then repository metadata (nearest ancestor `.git`/`.codegraph`), then the +project root. `cacheLocation` accepts `"project"` (anchor at `projectRoot`), `"user"` (anchor at +the platform user cache directory), `"repo"` (the default repository-metadata search), or an +absolute path. Unlike `cacheDir`, an absolute `cacheLocation` is an anchor, not the final cache +directory: the resolved cache lives in a project-namespaced subdirectory underneath it, since one +anchor can be shared by multiple projects. + +`createAgentSession()` (when `useConfig` is not disabled) and `createCodeReviewSession()` both +read `cache.location` from `codegraph.config.json` (project config, falling back to user +config) the same way they merge `discovery`, `graph`, and `languages.extensions`; an explicit +`buildOptions.cacheLocation`/`cacheDir` always takes precedence over the config value. + ## Project file discovery and graph building `listProjectFiles` defaults to source files plus common project manifests and lockfiles across supported languages, for example `package.json`, `requirements.txt`, `pyproject.toml`, and `Cargo.toml`. diff --git a/src/agent/renamePreview.ts b/src/agent/renamePreview.ts index b6e2caee..99999752 100644 --- a/src/agent/renamePreview.ts +++ b/src/agent/renamePreview.ts @@ -525,7 +525,11 @@ async function addScopeConflicts( const file = normalizeAgentFilePath(snapshot.root, target.file); let localCollision: SymbolDef | undefined; try { - const parsed = await ensureParsedContext(target.file, snapshot.index.parsed?.get(fileIdentityKey(target.file))); + const parsed = await ensureParsedContext( + target.file, + snapshot.index.parsed?.get(fileIdentityKey(target.file)), + snapshot.index.languageExtensions, + ); const scopeIndex = getCachedScope(snapshot.index, target.file, moduleIndex, parsed); const targetBinding = scopeIndex.all.find( (binding) => @@ -604,6 +608,7 @@ async function addScopeConflicts( const parsed = await ensureParsedContext( reference.file, snapshot.index.parsed?.get(fileIdentityKey(reference.file)), + snapshot.index.languageExtensions, ); const scopeIndex = getCachedScope(snapshot.index, reference.file, consumer, parsed); const activeBinding = scopeIndex.all.find((binding) => binding.import === activeImport); @@ -783,7 +788,11 @@ async function collectTextualRenameEdits( if (!loaded) continue; let parsed; try { - parsed = await ensureParsedContext(file, input.snapshot.index.parsed?.get(fileIdentityKey(file))); + parsed = await ensureParsedContext( + file, + input.snapshot.index.parsed?.get(fileIdentityKey(file)), + input.snapshot.index.languageExtensions, + ); } catch (error: unknown) { input.unsafeSites.push({ location: { file: loaded.displayPath, range: zeroRange() }, diff --git a/src/agent/session.ts b/src/agent/session.ts index 3ac12602..e47a8427 100644 --- a/src/agent/session.ts +++ b/src/agent/session.ts @@ -99,6 +99,7 @@ type AgentDiscoverySettings = { discoveryOptions?: ProjectFileDiscoveryOptions; graphOptions?: BuildOptions["graph"]; languageExtensions?: BuildOptions["languageExtensions"]; + cacheLocation?: BuildOptions["cacheLocation"]; }; type AgentSessionFilePlan = AgentDiscoverySettings & { @@ -120,6 +121,7 @@ async function resolveAgentDiscoverySettings(options: AgentSessionOptions): Prom const graphOptions = config.graph || options.buildOptions?.graph ? graph : undefined; const languageExtensions = normalizeLanguageExtensions(options.buildOptions?.languageExtensions) ?? config.languages?.extensions; + const cacheLocation = options.buildOptions?.cacheLocation ?? config.cache?.location; const discoveryOptions = hasDiscoveryOptions(discovery) ? { ...discovery, globRoot: discovery.globRoot ?? options.root } : undefined; @@ -127,11 +129,13 @@ async function resolveAgentDiscoverySettings(options: AgentSessionOptions): Prom ...(discoveryOptions ? { discoveryOptions } : {}), ...(graphOptions ? { graphOptions } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; } async function resolveAgentSessionFilePlan(options: AgentSessionOptions): Promise { - const { discoveryOptions, graphOptions, languageExtensions } = await resolveAgentDiscoverySettings(options); + const { discoveryOptions, graphOptions, languageExtensions, cacheLocation } = + await resolveAgentDiscoverySettings(options); // Prefer the manifest-plus-Git reconciliation over a full recursive scan whenever it // can be trusted. Preserve its changed/untracked evidence so the indexer does not // repeat the same Git subprocesses immediately afterward. @@ -140,6 +144,7 @@ async function resolveAgentSessionFilePlan(options: AgentSessionOptions): Promis ...(discoveryOptions ? { discovery: discoveryOptions } : {}), ...(graphOptions ? { graph: graphOptions } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; const incrementalPlan = await resolveIncrementalFilePlan(options.root, incrementalOptions); if (incrementalPlan) { @@ -149,6 +154,7 @@ async function resolveAgentSessionFilePlan(options: AgentSessionOptions): Promis ...(discoveryOptions ? { discoveryOptions } : {}), ...(graphOptions ? { graphOptions } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; } const { DEFAULT_PROJECT_PATTERNS } = await import("../util/projectFiles.js"); @@ -160,6 +166,7 @@ async function resolveAgentSessionFilePlan(options: AgentSessionOptions): Promis ...(discoveryOptions ? { discoveryOptions } : {}), ...(graphOptions ? { graphOptions } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; } @@ -324,7 +331,8 @@ export function createAgentSession(options: AgentSessionOptions): AgentSession { const loadBase = async (): Promise => { if (cachedBase) return cachedBase; const loadPromise = (async () => { - const { files, discoveryOptions, graphOptions, languageExtensions, incrementalPlan } = await loadFilePlan(); + const { files, discoveryOptions, graphOptions, languageExtensions, cacheLocation, incrementalPlan } = + await loadFilePlan(); const buildOptions: IncrementalBuildOptions = { ...options.buildOptions, ...(graphOptions ? { graph: graphOptions } : {}), @@ -341,6 +349,7 @@ export function createAgentSession(options: AgentSessionOptions): AgentSession { : {}), ...(discoveryOptions ? { discovery: discoveryOptions } : {}), ...(languageExtensions ? { languageExtensions } : {}), + ...(cacheLocation ? { cacheLocation } : {}), }; if (options.buildOptions?.useNativeWorkers === undefined && files.length >= NATIVE_WORKER_AUTO_FILE_THRESHOLD) { buildOptions.useNativeWorkers = true; @@ -375,11 +384,12 @@ export function createAgentSession(options: AgentSessionOptions): AgentSession { const loadSymbolGraph = async (base: AgentProjectBaseSnapshot): Promise => { if (cachedSymbolGraph) return cachedSymbolGraph; const loadPromise = (async () => { - const { graphOptions } = await loadFilePlan(); + const { graphOptions, cacheLocation } = await loadFilePlan(); const cacheOptions: BuildOptions = { ...options.buildOptions, ...(graphOptions ? { graph: graphOptions } : {}), cache: options.buildOptions?.cache ?? "disk", + ...(cacheLocation ? { cacheLocation } : {}), }; const persisted = await tryLoadDetailedSymbolGraphSnapshot(options.root, cacheOptions, base.index); if (persisted) return persisted; diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index e663a1c5..420b98a5 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1,4 +1,5 @@ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveCacheLocation } from "../indexer/build-cache/location.js"; import { @@ -320,15 +321,15 @@ export function findStaleNpmRetirementPaths(packageRoot: string, limit = 20): st } } /** - * Best-effort, dependency-light read of `cache.location` from `codegraph.config.json` in the - * current working directory. Deliberately avoids importing `../config.js` (which pulls in zod - * and the full config schema): doctor is a fast, low-dependency health check, and the eager + * Best-effort, dependency-light read of `cache.location`, mirroring `loadCodegraphConfig`'s + * project-over-user precedence without importing `../config.js` (which pulls in zod and the + * full config schema): doctor is a fast, low-dependency health check, and the eager * dist-module-loading budget in `cli-startup-eager-modules.test.ts` enforces that. Malformed or * missing config is silently ignored; the fuller schema validation still applies to real builds. */ -function readProjectCacheLocation(root: string): string | undefined { +function readCacheLocationField(configPath: string): string | undefined { try { - const raw = fs.readFileSync(path.join(root, "codegraph.config.json"), "utf8"); + const raw = fs.readFileSync(configPath, "utf8"); const parsed = JSON.parse(raw) as { cache?: { location?: unknown } }; const location = parsed.cache?.location; return typeof location === "string" && location.trim() ? location.trim() : undefined; @@ -337,13 +338,25 @@ function readProjectCacheLocation(root: string): string | undefined { } } +function readUserCacheLocation(): string | undefined { + const configRoot = + process.platform === "win32" + ? process.env.APPDATA?.trim() || path.join(os.homedir(), "AppData", "Roaming") + : process.env.XDG_CONFIG_HOME?.trim() || path.join(os.homedir(), ".config"); + return readCacheLocationField(path.join(configRoot, "codegraph", "config.json")); +} + +function readEffectiveCacheLocation(root: string): string | undefined { + return readCacheLocationField(path.join(root, "codegraph.config.json")) ?? readUserCacheLocation(); +} + export function buildDoctorReport(indexPath?: string): DoctorReport { const packageIdentity = getCodegraphPackageIdentity(); const loadError = getNativeTreeSitterLoadError(); const origin = getNativeBindingOrigin(); const runtimeIdentity = captureCodegraphRuntimeIdentity(origin); const update = createInstalledVersionChecker(runtimeIdentity, { warn: () => undefined }).check(true); - const cacheLocation = readProjectCacheLocation(process.cwd()); + const cacheLocation = readEffectiveCacheLocation(process.cwd()); const cacheResolution = resolveCacheLocation(process.cwd(), cacheLocation ? { cacheLocation } : undefined); return { package: packageIdentity, diff --git a/src/impact/callCompatibility.ts b/src/impact/callCompatibility.ts index 5c7c6e38..33620d1a 100644 --- a/src/impact/callCompatibility.ts +++ b/src/impact/callCompatibility.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { findReferences, goToDefinition } from "../indexer/navigation.js"; -import { ensureParsedContext } from "../indexer/parse-context.js"; +import { ensureParsedContext, type ParsedFileContext } from "../indexer/parse-context.js"; import { SymbolKind, type ProjectIndex, type Reference, type SymbolDef } from "../indexer/types.js"; import { supportForFile } from "../languages.js"; import { isJsTsLanguage } from "../languages/js-family.js"; @@ -929,7 +929,12 @@ async function collectVerifiedCallsiteReferences( if (!support || !supportsCallCompatibilityLanguage(support.id)) { continue; } - const parsed = await tryEnsureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), diagnostics); + const parsed = await tryEnsureParsedContext( + file, + index.parsed?.get(fileIdentityKey(file)), + index.languageExtensions, + diagnostics, + ); if (!parsed) { continue; } @@ -991,17 +996,16 @@ function classifyCompatibility( return { status: "compatible", reason: "compatible_argument_count" }; } -async function tryEnsureParsedContext( +function tryEnsureParsedContext( file: string, parsedEntry: Parameters[1], + languageExtensions: Parameters[2], diagnostics: ImpactDiagnostics["callCompatibility"] | undefined, -): Promise> | null> { - try { - return await ensureParsedContext(file, parsedEntry); - } catch { +): Promise { + return ensureParsedContext(file, parsedEntry, languageExtensions).catch(() => { incrementSkippedReason(diagnostics, "parse-failed"); return null; - } + }); } function incrementSkippedReason(diagnostics: ImpactDiagnostics["callCompatibility"] | undefined, reason: string): void { @@ -1035,6 +1039,7 @@ async function buildCallCompatibilityHintForReference(input: { const parsedCallsite = await tryEnsureParsedContext( ref.file, index.parsed?.get(fileIdentityKey(ref.file)), + index.languageExtensions, diagnostics, ); if (!parsedCallsite) { @@ -1113,6 +1118,7 @@ export async function attachCallCompatibilityHints( const parsedDefinition = await tryEnsureParsedContext( changedSymbol.file, index.parsed?.get(fileIdentityKey(changedSymbol.file)), + index.languageExtensions, diagnostics, ); if (!parsedDefinition) { diff --git a/src/impact/map.ts b/src/impact/map.ts index 660e7212..136d3525 100644 --- a/src/impact/map.ts +++ b/src/impact/map.ts @@ -49,7 +49,7 @@ export async function locateChangedSymbolsWithLines( let parsedEntry; try { - parsedEntry = await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file))); + parsedEntry = await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), index.languageExtensions); } catch { return { changedSymbols: [], changedLines, parseFailed: true }; } @@ -286,7 +286,7 @@ export async function mapChangedLinesToSymbols( let parsedEntry; try { - parsedEntry = await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file))); + parsedEntry = await ensureParsedContext(file, index.parsed?.get(fileIdentityKey(file)), index.languageExtensions); } catch { return new Map(); } diff --git a/src/impact/referenceCache.ts b/src/impact/referenceCache.ts index 27969b77..f9b8ae99 100644 --- a/src/impact/referenceCache.ts +++ b/src/impact/referenceCache.ts @@ -108,7 +108,7 @@ async function attachReferenceContext( let cached = perFileCache.get(fileKey); if (!cached) { const parsedEntry = index.parsed?.get(fileKey); - const parsed = await ensureParsedContext(ref.file, parsedEntry); + const parsed = await ensureParsedContext(ref.file, parsedEntry, index.languageExtensions); cached = { source: parsed.source, tree: parsed.tree, sup: parsed.sup }; perFileCache.set(fileKey, cached); } diff --git a/src/impact/suggestions.ts b/src/impact/suggestions.ts index fb1db426..0e22dbfe 100644 --- a/src/impact/suggestions.ts +++ b/src/impact/suggestions.ts @@ -70,7 +70,11 @@ export async function collectImpactSuggestions( pushUniqueSuggestion(output, seen, suggestion); } - const parsedEntry = await ensureParsedContext(absoluteFile, index.parsed?.get(fileIdentityKey(absoluteFile))); + const parsedEntry = await ensureParsedContext( + absoluteFile, + index.parsed?.get(fileIdentityKey(absoluteFile)), + index.languageExtensions, + ); if (!parsedEntry) continue; const changedLines = collectChangedLines(fileChange.hunks); diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index bc0f385a..6bd8b1bc 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -177,12 +177,19 @@ export function sanitizeManifestEntriesForRoot( return sanitizedEntries; } -export function sanitizeManifestTransientFilesForRoot(projectRoot: string, files: unknown): string[] { +export function sanitizeManifestTransientFilesForRoot( + projectRoot: string, + storedProjectRoot: string, + files: unknown, +): string[] { if (!Array.isArray(files)) return []; const sanitizedFiles = new Set(); for (const value of files) { if (typeof value !== "string") continue; - const file = path.resolve(projectRoot, value).replace(/\\/g, "/"); + const storedFile = cacheAbsolutePath(storedProjectRoot, value); + if (!isFilePathWithinRoot(storedProjectRoot, storedFile)) continue; + const relativeFile = cacheRelativePath(storedProjectRoot, storedFile); + const file = cacheAbsolutePath(projectRoot, relativeFile); if (isFilePathWithinRoot(projectRoot, file)) sanitizedFiles.add(file); } return [...sanitizedFiles]; @@ -308,7 +315,7 @@ export async function loadManifest(projectRoot: string, opts?: BuildOptions): Pr ...parsed, version: MANIFEST_VERSION, files: transformManifestEntries(projectRoot, relativeFiles, false), - transientFiles: sanitizeManifestTransientFilesForRoot(projectRoot, parsed.transientFiles), + transientFiles: sanitizeManifestTransientFilesForRoot(projectRoot, parsed.projectRoot, parsed.transientFiles), ...(symlinkDirectories !== undefined ? { symlinkDirectories } : {}), }; return migrated; diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 61b83c6b..8023db90 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -1241,9 +1241,11 @@ export async function buildProjectIndexIncremental( opts?.additionalFiles ?? [], "Additional index file", ); - const previousTransientFiles = sanitizeManifestTransientFilesForRoot(projectRoot, manifest.transientFiles).filter( - (file) => Object.hasOwn(trackedEntries, file), - ); + const previousTransientFiles = sanitizeManifestTransientFilesForRoot( + projectRoot, + projectRoot, + manifest.transientFiles, + ).filter((file) => Object.hasOwn(trackedEntries, file)); const previousTransientFileSet = new Set(previousTransientFiles); const needsGitScan = !!opts?.gitBase || !!opts?.changedSince; const gitFiles = needsGitScan ? await listChangedFiles(projectRoot, buildIncrementalGitDiffOptions(opts)) : []; diff --git a/src/indexer/navigation-goto.ts b/src/indexer/navigation-goto.ts index bed87359..66e6177b 100644 --- a/src/indexer/navigation-goto.ts +++ b/src/indexer/navigation-goto.ts @@ -169,7 +169,7 @@ export async function resolveMemberAccessDefinition(params: { const objDef = await resolveReceiverDefinition(obj, source, sup, resolveExpression); if (objDef) { - const targetContext = await ensureParsedContext(objDef.file); + const targetContext = await ensureParsedContext(objDef.file, undefined, index.languageExtensions); const start = objDef.range.start; const targetPosition = { row: start.line - 1, @@ -529,7 +529,7 @@ async function resolveMemberDefinitionForBase( baseDef: SymbolDef, member: string, ): Promise { - const targetContext = await ensureParsedContext(baseDef.file); + const targetContext = await ensureParsedContext(baseDef.file, undefined, index.languageExtensions); const start = baseDef.range.start; const targetPosition = { row: start.line - 1, diff --git a/src/indexer/navigation-references.ts b/src/indexer/navigation-references.ts index 91eeafc5..1cde114d 100644 --- a/src/indexer/navigation-references.ts +++ b/src/indexer/navigation-references.ts @@ -132,6 +132,7 @@ export async function buildPhpQualifiedNames( const definitionParsed = await ensureParsedContext( definitionFile, index.parsed?.get(fileIdentityKey(definitionFile)), + index.languageExtensions, ); if (definitionParsed.sup.id !== "php") { return []; @@ -152,7 +153,7 @@ async function collectNamedNodeReferences( ): Promise<{ ranges: Range[]; parsed: ParsedFileContext } | null> { try { const parsedEntry = index.parsed?.get(fileIdentityKey(fileId)); - const parsed = await ensureParsedContext(fileId, parsedEntry); + const parsed = await ensureParsedContext(fileId, parsedEntry, index.languageExtensions); const identifierTypes = new Set([ ...parsed.sup.nodeTypes.identifier, ...(parsed.sup.nodeTypes.propertyIdentifier ?? []), diff --git a/src/indexer/navigation.ts b/src/indexer/navigation.ts index 739b9ae9..36ef940d 100644 --- a/src/indexer/navigation.ts +++ b/src/indexer/navigation.ts @@ -1,4 +1,4 @@ -import { type LanguageSupport } from "../languages.js"; +import { type LanguageExtensionMap, type LanguageSupport } from "../languages.js"; import type { SyntaxNodeLike, SyntaxTreeLike } from "../languages/types.js"; import { ensureParsedContext, type ParsedFileContext } from "./parse-context.js"; import { resolveMemberAccessDefinition, supportsReceiverMemberResolution } from "./navigation-goto.js"; @@ -342,7 +342,13 @@ export async function findReferences( : fileIdentityKey(targetFile) === fileIdentityKey(definitionFile); if (!matchesDef) continue; const parsed = await ensureCandidateParsed(); - const ranges = await collectNamespaceMemberRefs(fileId, imp.localNS, exportedName, parsed); + const ranges = await collectNamespaceMemberRefs( + fileId, + imp.localNS, + exportedName, + parsed, + index.languageExtensions, + ); for (const range of ranges) { if (hasReachedMaxReferences()) break; pushRef({ @@ -540,8 +546,9 @@ export async function collectNamespaceMemberRefs( ns: string, member: string, parsedContext?: ParsedFileContext, + languageExtensions?: LanguageExtensionMap, ): Promise { - const parsed = parsedContext ?? (await ensureParsedContext(file, undefined)); + const parsed = parsedContext ?? (await ensureParsedContext(file, undefined, languageExtensions)); const sup = parsed.sup; const source = parsed.source; const tree = parsed.tree; diff --git a/src/indexer/types.ts b/src/indexer/types.ts index 37ac0ed0..b47854d5 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -142,7 +142,15 @@ export type BuildOptions = { onProgress?: ((progress: ProgressUpdate) => void) | undefined; threads?: number; cache?: "off" | "memory" | "disk"; + /** Explicit disk-cache directory. Highest-precedence anchor; also settable via `CODEGRAPH_CACHE_DIR`. */ cacheDir?: string; + /** + * Disk-cache anchor when `cacheDir`/`CODEGRAPH_CACHE_DIR` are unset: `"project"` anchors at + * `projectRoot`, `"user"` anchors at the platform user cache directory, `"repo"` (or omitted) + * searches ancestor directories for repository metadata and falls back to `projectRoot`. Any + * other value is treated as an absolute anchor directory (like `cacheDir`), not a final cache + * path: the resolved cache lives in a namespaced subdirectory under the anchor. + */ cacheLocation?: CacheLocation; cacheStrict?: boolean; useBloomFilters?: boolean; diff --git a/src/indexer/workspace-symbols.ts b/src/indexer/workspace-symbols.ts index 15b95b26..d2e7b01c 100644 --- a/src/indexer/workspace-symbols.ts +++ b/src/indexer/workspace-symbols.ts @@ -167,7 +167,11 @@ async function buildImportCandidates(index: ProjectIndex): Promise { expect(buildSpy.mock.calls[0]?.[1]?.languageExtensions).toEqual({ ".custom": "ts" }); }); + it("merges codegraph.config.json cache.location into incremental agent build options", async () => { + const root = await mkRepo(); + const cacheLocation = path.join(root, "custom-cache"); + await fs.writeFile( + path.join(root, "codegraph.config.json"), + JSON.stringify({ cache: { location: cacheLocation } }), + ); + const buildSpy = vi.spyOn(indexerBuild, "buildProjectIndexIncremental"); + + await createAgentSession({ root }).loadProject({ symbolGraph: "skip" }); + + expect(buildSpy.mock.calls[0]?.[1]?.cacheLocation).toBe(cacheLocation); + }); + + it("prefers an explicit buildOptions.cacheLocation over codegraph.config.json", async () => { + const root = await mkRepo(); + const configCacheLocation = path.join(root, "config-cache"); + const explicitCacheLocation = path.join(root, "explicit-cache"); + await fs.writeFile( + path.join(root, "codegraph.config.json"), + JSON.stringify({ cache: { location: configCacheLocation } }), + ); + const buildSpy = vi.spyOn(indexerBuild, "buildProjectIndexIncremental"); + + await createAgentSession({ root, buildOptions: { cacheLocation: explicitCacheLocation } }).loadProject({ + symbolGraph: "skip", + }); + + expect(buildSpy.mock.calls[0]?.[1]?.cacheLocation).toBe(explicitCacheLocation); + }); + it("uses programmatic language extensions when listing agent session files", async () => { const root = await mkRepo(); await fs.writeFile(path.join(root, "feature.custom"), "export const customFeature = 1;\n"); diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 00bb1852..d33c3097 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -2649,4 +2649,47 @@ describe("Cache invalidation and strict hashing", () => { expect(index.projectRoot).toBe(normalize(path.resolve(root))); }); + + it("rebases legacy v3 absolute transientFiles from the stored root after a project move", async () => { + const sourceRoot = await mkTmpDir("dg-manifest-transient-move-source-"); + const movedRoot = `${sourceRoot}-moved`; + await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export const entry = 1;\n", "utf8"); + await fsp.writeFile(path.join(sourceRoot, ".gitignore"), "outside/\n", "utf8"); + const outsideFile = path.join(sourceRoot, "outside", "extra.ts"); + await fsp.mkdir(path.dirname(outsideFile), { recursive: true }); + await fsp.writeFile(outsideFile, "export const extra = 1;\n", "utf8"); + + await buildProjectIndexIncremental(sourceRoot, { cache: "disk", threads: 1, additionalFiles: [outsideFile] }); + const manifestPath = manifestPathFor(sourceRoot); + const manifest = JSON.parse(await fsp.readFile(manifestPath, "utf8")) as { + version: number; + transientFiles?: string[]; + }; + expect(manifest.transientFiles).toEqual(["outside/extra.ts"]); + // Simulate a genuine legacy v3 manifest, which persisted transientFiles as absolute paths. + manifest.version = 3; + manifest.transientFiles = [normalize(outsideFile)]; + await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8"); + + await fsp.rename(sourceRoot, movedRoot); + const movedOutsideFile = path.join(movedRoot, "outside", "extra.ts"); + // Force a genuine content change so the incremental diff/manifest-rewrite path runs + // instead of the whole-snapshot fast path (which leaves manifest.json untouched when + // nothing changed and would otherwise mask this migration). + await fsp.writeFile(path.join(movedRoot, "entry.ts"), "export const entry = 2;\n", "utf8"); + + const rebuilt = await buildProjectIndexIncremental(movedRoot, { + cache: "disk", + threads: 1, + additionalFiles: [movedOutsideFile], + }); + + expect(rebuilt.byFile.has(fileIdentityKey(normalize(movedOutsideFile)))).toBe(true); + const rebuiltManifest = JSON.parse(await fsp.readFile(manifestPathFor(movedRoot), "utf8")) as { + version: number; + transientFiles?: string[]; + }; + expect(rebuiltManifest.version).toBe(MANIFEST_VERSION); + expect(rebuiltManifest.transientFiles).toEqual(["outside/extra.ts"]); + }); }); diff --git a/tests/cli-command-modules.test.ts b/tests/cli-command-modules.test.ts index 67ab6e54..5c7a2f9f 100644 --- a/tests/cli-command-modules.test.ts +++ b/tests/cli-command-modules.test.ts @@ -1405,6 +1405,32 @@ describe("CLI command modules", () => { } }); + test("doctor falls back to the platform user config when project config has no cache.location", async () => { + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-user-cache-config-")); + const userConfigRoot = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-user-config-root-")); + const cacheLocation = path.join(tempDir, "user-custom-cache"); + await fsp.mkdir(path.join(userConfigRoot, "codegraph"), { recursive: true }); + await fsp.writeFile( + path.join(userConfigRoot, "codegraph", "config.json"), + JSON.stringify({ cache: { location: cacheLocation } }), + "utf8", + ); + const previousCwd = process.cwd(); + process.chdir(tempDir); + vi.stubEnv("APPDATA", userConfigRoot); + vi.stubEnv("XDG_CONFIG_HOME", userConfigRoot); + try { + const report = buildDoctorReport(); + expect(report.cache.layer).toBe("explicit"); + expect(report.cache.anchor).toBe(cacheLocation.replace(/\\/g, "/")); + } finally { + process.chdir(previousCwd); + vi.unstubAllEnvs(); + await fsp.rm(tempDir, { recursive: true, force: true }); + await fsp.rm(userConfigRoot, { recursive: true, force: true }); + } + }); + test("builds doctor reports for explicit index artifact paths", async () => { const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-module-")); const artifactPath = path.join(tempDir, "codegraph.json"); diff --git a/tests/workspace-symbols.test.ts b/tests/workspace-symbols.test.ts index 636b5b9d..202a10d7 100644 --- a/tests/workspace-symbols.test.ts +++ b/tests/workspace-symbols.test.ts @@ -278,6 +278,34 @@ describe("workspace symbol lookup", () => { } }); + it("resolves import bindings in files with a custom configured extension", async () => { + const isolatedRoot = await fs.mkdtemp(path.join(os.tmpdir(), "codegraph-workspace-symbols-customext-")); + try { + const sourceDir = path.join(isolatedRoot, "src"); + const targetFile = path.join(sourceDir, "service.ts"); + const importerFile = path.join(sourceDir, "importer.mylang"); + await fs.mkdir(sourceDir, { recursive: true }); + await fs.writeFile(targetFile, "export class Service {}\n"); + await fs.writeFile( + importerFile, + "import { Service as LocalService } from './service';\nexport function useService() { return LocalService; }\n", + ); + const languageExtensions = { ".mylang": "ts" }; + const isolatedIndex = await buildProjectIndexFromFiles(isolatedRoot, [targetFile, importerFile], { + cache: "off", + languageExtensions, + }); + + const result = await workspaceSymbols(isolatedIndex, { query: "LocalService", includeImports: true }); + + expect(result.importScanFailures).toBe(0); + expect(result.symbols).toHaveLength(1); + expect(result.symbols[0]).toMatchObject({ name: "LocalService", imported: true }); + } finally { + await fs.rm(isolatedRoot, { recursive: true, force: true }); + } + }); + it("reports failed import scans and omitted aliases instead of silently dropping them", async () => { const importerEntry = [...index.byFile.entries()].find(([file]) => file.endsWith("/src/importer.ts")); expect(importerEntry).toBeDefined(); From b98fdb15416644f94930c5094a78a0da0b0a140a Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 01:34:54 -0400 Subject: [PATCH 25/28] fix: address fourth batch of suppressed review findings - util/projectFiles.ts: keep default ignores active during the symlink probe scan for any root an include glob does not actually mention, instead of dropping every default ignore whenever any include glob is present. Large ignored trees (node_modules, vendor, build) no longer get walked just because an unrelated include glob was set. - agent/query-index/store.ts: give each search term its own bounded prefetch budget in candidateChunksForTerms instead of one shared, path-ordered budget. A common term matching thousands of early-path chunks could previously exhaust the whole budget before a rarer term's match (or a chunk matching multiple terms) later in path order was ever read, so the in-memory scorer could never see it. - indexer/build-index.ts: skip per-file module cache reuse on the warm (non-incremental) build path when resolveNodeModules just turned on, matching the incremental path (which already force-marks every file changed for this mode). A cached ModuleIndex's ImportBinding.resolved values are computed under whatever resolveNodeModules state was active when the file was cached; reusing them returns stale/unresolved node-module import targets even though graph-edge reuse was already disabled for this mode. - util/sqliteSchema.ts: only invoke migrateTable when the on-disk schema is genuinely behind (missing or older), not on every open at the already-current version. module-cache.ts's and unitCache.ts's migrate callbacks both do an O(rows) relative-path backfill scan, which previously ran on every warm cache open regardless of whether there was anything to migrate. - cli/doctor.ts: validate cache.location the same way the real Zod schema does ("project"/"repo"/"user"/absolute path) before treating a project or user config value as effective, so doctor does not report a relative (schema-invalid) value as a working explicit cache path. - indexer/types.ts + docs/library-api.md: describe cacheDir and CODEGRAPH_CACHE_DIR consistently as namespaced anchors (not final cache directories), matching how an absolute cacheLocation already was documented. Adds regression coverage in query-index.test.ts (per-term prefetch fairness), node-modules-and-paths.test.ts (stale resolveNodeModules cache reuse, verified to fail without the fix), sqlite-common.test.ts (migrateTable invocation count across repeated opens), and cli-command-modules.test.ts (doctor rejecting an invalid relative cache.location). --- docs/library-api.md | 15 ++--- src/agent/query-index/store.ts | 93 +++++++++++++++------------- src/cli/doctor.ts | 8 ++- src/indexer/build-index.ts | 9 ++- src/indexer/types.ts | 6 +- src/util/projectFiles.ts | 27 +++++++- src/util/sqliteSchema.ts | 5 +- tests/cli-command-modules.test.ts | 19 ++++++ tests/node-modules-and-paths.test.ts | 27 ++++++++ tests/query-index.test.ts | 24 +++++++ tests/sqlite-common.test.ts | 33 ++++++++++ 11 files changed, 209 insertions(+), 57 deletions(-) diff --git a/docs/library-api.md b/docs/library-api.md index 43fbf5ab..baf35f8d 100644 --- a/docs/library-api.md +++ b/docs/library-api.md @@ -630,13 +630,14 @@ const incremental = await buildProjectIndexIncremental(root, { (`cache: "disk"`) is written; they have no effect for `cache: "memory"` or `"off"`. Persisted cache contents store project-relative paths, so a cache can be moved along with its project. -Anchor selection precedence: `cacheDir` (an explicit final directory), then `CODEGRAPH_CACHE_DIR`, -then `cacheLocation`, then repository metadata (nearest ancestor `.git`/`.codegraph`), then the -project root. `cacheLocation` accepts `"project"` (anchor at `projectRoot`), `"user"` (anchor at -the platform user cache directory), `"repo"` (the default repository-metadata search), or an -absolute path. Unlike `cacheDir`, an absolute `cacheLocation` is an anchor, not the final cache -directory: the resolved cache lives in a project-namespaced subdirectory underneath it, since one -anchor can be shared by multiple projects. +Anchor selection precedence: `cacheDir`, then `CODEGRAPH_CACHE_DIR`, then `cacheLocation`, then +repository metadata (nearest ancestor `.git`/`.codegraph`), then the project root. `cacheLocation` +accepts `"project"` (anchor at `projectRoot`), `"user"` (anchor at the platform user cache +directory), `"repo"` (the default repository-metadata search), or an absolute path. None of +`cacheDir`, `CODEGRAPH_CACHE_DIR`, or an absolute `cacheLocation` is the final cache directory: +each is an anchor, and the resolved cache lives in a project-namespaced subdirectory underneath +it, since one anchor can be shared by multiple projects. The project root's own default location +(no anchor configured) is the one exception, since only one project can occupy it. `createAgentSession()` (when `useConfig` is not disabled) and `createCodeReviewSession()` both read `cache.location` from `codegraph.config.json` (project config, falling back to user diff --git a/src/agent/query-index/store.ts b/src/agent/query-index/store.ts index 773d81a7..49202bad 100644 --- a/src/agent/query-index/store.ts +++ b/src/agent/query-index/store.ts @@ -329,52 +329,57 @@ export class QueryIndexStore { ): StoredQueryIndexChunk[] { if (!terms.length || !paths.length) return []; const normalizedLimit = normalizedCandidateLimit(limit); - const ftsTerms = terms.filter((term) => codePointLength(term) >= 3); - const directTerms = terms.filter((term) => codePointLength(term) < 3); - const conditions: string[] = []; - const directParameters: string[] = []; - if (ftsTerms.length) { - conditions.push("chunks.chunk_id IN (SELECT rowid FROM fts_matches)"); - } - for (const term of directTerms) { - conditions.push("instr(chunks.normalized_text, ?) > 0"); - directParameters.push(term); - } - for (const term of terms) { - conditions.push("instr(replace(chunks.normalized_text, ' ', ''), ?) > 0"); - directParameters.push(term); - } - const ftsQuery = ftsTerms.map(escapeFtsTrigramTerm).join(" OR "); - const prefix = ftsTerms.length - ? "WITH fts_matches AS (SELECT rowid FROM chunk_search WHERE chunk_search MATCH ?)" - : ""; + // Bound each term independently instead of sharing one global, path-ordered budget: + // a common term matching thousands of early-path chunks would otherwise exhaust the + // budget before a rarer term's (or a multi-term) match later in path order is read. + const perTermLimit = Math.max(1, Math.ceil(normalizedLimit / terms.length)); const candidates = new Map(); const batchSize = 500; - for (let offset = 0; offset < paths.length && candidates.size < normalizedLimit; offset += batchSize) { - const batch = paths.slice(offset, offset + batchSize); - const placeholders = batch.map(() => "?").join(", "); - const remaining = normalizedLimit - candidates.size; - const rows = this.db - .prepare( - ` - ${prefix} - SELECT files.path AS path, chunks.ordinal, chunks.kind, chunks.name, - chunks.start_line, chunks.end_line, chunks.text, chunks.normalized_text - FROM chunks - JOIN files ON files.file_id = chunks.file_id - WHERE files.path IN (${placeholders}) - AND (${conditions.join(" OR ")}) - ORDER BY files.path, chunks.ordinal - LIMIT ? - `, - ) - .all( - ...(ftsTerms.length ? [ftsQuery, ...batch, ...directParameters] : [...batch, ...directParameters]), - remaining, - ) as Array>; - for (const row of rows) { - const chunk = storedCandidateChunkFromRow(row); - if (chunk) candidates.set(`${chunk.path}\0${chunk.ordinal}`, chunk); + for (const term of terms) { + const isFtsEligible = codePointLength(term) >= 3; + const conditions: string[] = []; + const parameters: string[] = []; + if (isFtsEligible) { + conditions.push("chunks.chunk_id IN (SELECT rowid FROM fts_matches)"); + } else { + conditions.push("instr(chunks.normalized_text, ?) > 0"); + parameters.push(term); + } + conditions.push("instr(replace(chunks.normalized_text, ' ', ''), ?) > 0"); + parameters.push(term); + const prefix = isFtsEligible + ? "WITH fts_matches AS (SELECT rowid FROM chunk_search WHERE chunk_search MATCH ?)" + : ""; + let termMatches = 0; + for (let offset = 0; offset < paths.length && termMatches < perTermLimit; offset += batchSize) { + const batch = paths.slice(offset, offset + batchSize); + const placeholders = batch.map(() => "?").join(", "); + const remaining = perTermLimit - termMatches; + const rows = this.db + .prepare( + ` + ${prefix} + SELECT files.path AS path, chunks.ordinal, chunks.kind, chunks.name, + chunks.start_line, chunks.end_line, chunks.text, chunks.normalized_text + FROM chunks + JOIN files ON files.file_id = chunks.file_id + WHERE files.path IN (${placeholders}) + AND (${conditions.join(" OR ")}) + ORDER BY files.path, chunks.ordinal + LIMIT ? + `, + ) + .all( + ...(isFtsEligible ? [escapeFtsTrigramTerm(term), ...batch, ...parameters] : [...batch, ...parameters]), + remaining, + ) as Array>; + for (const row of rows) { + const chunk = storedCandidateChunkFromRow(row); + if (!chunk) continue; + const key = `${chunk.path}\0${chunk.ordinal}`; + if (!candidates.has(key)) termMatches += 1; + candidates.set(key, chunk); + } } } return [...candidates.values()]; diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 420b98a5..2cc357f0 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -327,12 +327,18 @@ export function findStaleNpmRetirementPaths(packageRoot: string, limit = 20): st * dist-module-loading budget in `cli-startup-eager-modules.test.ts` enforces that. Malformed or * missing config is silently ignored; the fuller schema validation still applies to real builds. */ +function isValidCacheLocationValue(location: string): boolean { + return location === "project" || location === "repo" || location === "user" || path.isAbsolute(location); +} + function readCacheLocationField(configPath: string): string | undefined { try { const raw = fs.readFileSync(configPath, "utf8"); const parsed = JSON.parse(raw) as { cache?: { location?: unknown } }; const location = parsed.cache?.location; - return typeof location === "string" && location.trim() ? location.trim() : undefined; + if (typeof location !== "string") return undefined; + const trimmed = location.trim(); + return trimmed && isValidCacheLocationValue(trimmed) ? trimmed : undefined; } catch { return undefined; } diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 8023db90..c9f69429 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -747,7 +747,14 @@ async function buildIndexFromFileListShared( if (initialManifestEntry) manifestEntries.set(file, initialManifestEntry); } const cacheSig = cacheEnabled ? await moduleCacheSignatureForFile(file, sigInfo, opts) : sigInfo.cacheSig; - let mod: ModuleIndex | null = cacheEnabled ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) : null; + // A cached ModuleIndex's ImportBinding.resolved values were computed under the + // resolveNodeModules state active at write time; reusing them when that state + // just turned on would return stale (unresolved) node-module import targets even + // though graph-edge reuse is already disabled for this mode above. + const canReuseModuleCache = cacheEnabled && !graphOptions.resolveNodeModules; + let mod: ModuleIndex | null = canReuseModuleCache + ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) + : null; if (mod && fileReport) { fileReport.cached = (fileReport.cached ?? 0) + 1; } diff --git a/src/indexer/types.ts b/src/indexer/types.ts index b47854d5..04dccb84 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -142,7 +142,11 @@ export type BuildOptions = { onProgress?: ((progress: ProgressUpdate) => void) | undefined; threads?: number; cache?: "off" | "memory" | "disk"; - /** Explicit disk-cache directory. Highest-precedence anchor; also settable via `CODEGRAPH_CACHE_DIR`. */ + /** + * Highest-precedence disk-cache anchor; also settable via `CODEGRAPH_CACHE_DIR`. Like an + * absolute `cacheLocation`, this is an anchor, not the final cache directory: the resolved + * cache lives in a namespaced subdirectory underneath it. + */ cacheDir?: string; /** * Disk-cache anchor when `cacheDir`/`CODEGRAPH_CACHE_DIR` are unset: `"project"` anchors at diff --git a/src/util/projectFiles.ts b/src/util/projectFiles.ts index 4f12d34e..e26b1faa 100644 --- a/src/util/projectFiles.ts +++ b/src/util/projectFiles.ts @@ -172,6 +172,20 @@ function normalizeGlobPattern(globPattern: string): string { return globPattern.trim().replace(/\\/g, "/"); } +/** + * Whether an include glob explicitly re-opens a default-ignored root, so a default ignore + * should stay active for every OTHER root the includes never mention. Compares literal + * (non-wildcard) path segments rather than attempting general glob-vs-glob intersection. + */ +function isIgnoreGlobReopenedByIncludes(ignoreGlob: string, includeGlobs: readonly string[]): boolean { + const literalSegments = ignoreGlob.split("/").filter((segment) => segment && !segment.includes("*")); + if (!literalSegments.length) return false; + return includeGlobs.some((includeGlob) => { + const includeSegments = includeGlob.split("/"); + return literalSegments.every((segment) => includeSegments.includes(segment)); + }); +} + function isLocationIndependentGlob(globPattern: string): boolean { return globPattern.startsWith("**/"); } @@ -340,8 +354,17 @@ export async function listProjectFiles( const fastGlobIgnoreGlobs = [...DEFAULT_PROJECT_FILE_IGNORES, ...translatedUserIgnoreGlobs]; // Include globs are an explicit request to re-open otherwise ignored roots. Probe // those roots for safe directory links before the later filter reapplies the default - // ignores, so an included link is not lost before it can be traversed. - const symlinkProbeIgnoreGlobs = includeGlobs.length ? translatedUserIgnoreGlobs : fastGlobIgnoreGlobs; + // ignores, so an included link is not lost before it can be traversed. Default ignores + // an include never mentions (e.g. node_modules when only src/** is included) stay + // active, so the probe does not walk unrelated large ignored trees. + const symlinkProbeIgnoreGlobs = includeGlobs.length + ? [ + ...translatedUserIgnoreGlobs, + ...DEFAULT_PROJECT_FILE_IGNORES.filter( + (ignoreGlob) => !isIgnoreGlobReopenedByIncludes(ignoreGlob, includeGlobs), + ), + ] + : fastGlobIgnoreGlobs; try { const useGitignore = options?.useGitignore ?? true; diff --git a/src/util/sqliteSchema.ts b/src/util/sqliteSchema.ts index 0bfeadc4..3e4f6bf3 100644 --- a/src/util/sqliteSchema.ts +++ b/src/util/sqliteSchema.ts @@ -81,7 +81,10 @@ export function ensureSqliteVersionedTableSchema(args: { (schemaVersion.status === "ok" && schemaVersion.version > args.schemaVersion) ) { recreateSqliteTable(args.db, args.tableName, args.createTable); - } else { + } else if (schemaVersion.status === "missing" || schemaVersion.version < args.schemaVersion) { + // Only pay for the migration (which can include an O(rows) backfill scan) when the + // on-disk schema is actually behind; an already-current schema needs no work every + // time the database is opened. args.migrateTable(args.db); } writeSqliteSchemaVersion(args.db, args.schemaVersionKey, args.schemaVersion); diff --git a/tests/cli-command-modules.test.ts b/tests/cli-command-modules.test.ts index 5c7a2f9f..abfdb1aa 100644 --- a/tests/cli-command-modules.test.ts +++ b/tests/cli-command-modules.test.ts @@ -1405,6 +1405,25 @@ describe("CLI command modules", () => { } }); + test("doctor ignores a relative cache.location that would fail real schema validation", async () => { + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-invalid-cache-config-")); + await fsp.writeFile( + path.join(tempDir, "codegraph.config.json"), + JSON.stringify({ cache: { location: "relative-cache-dir" } }), + "utf8", + ); + const previousCwd = process.cwd(); + process.chdir(tempDir); + try { + const report = buildDoctorReport(); + expect(report.cache.layer).not.toBe("explicit"); + expect(report.cache.anchor).not.toContain("relative-cache-dir"); + } finally { + process.chdir(previousCwd); + await fsp.rm(tempDir, { recursive: true, force: true }); + } + }); + test("doctor falls back to the platform user config when project config has no cache.location", async () => { const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-user-cache-config-")); const userConfigRoot = await fsp.mkdtemp(path.join(os.tmpdir(), "codegraph-doctor-user-config-root-")); diff --git a/tests/node-modules-and-paths.test.ts b/tests/node-modules-and-paths.test.ts index b325cc39..7e2dfba8 100644 --- a/tests/node-modules-and-paths.test.ts +++ b/tests/node-modules-and-paths.test.ts @@ -5,6 +5,7 @@ import { buildProjectIndex, buildProjectIndexIncremental, type BuildReport } fro import { resolveFromNodeModules } from "../src/util/resolution/node.js"; import { collectGraph } from "../src/index.js"; import { mkTmpDir, normalizeTestPath } from "./helpers/filesystem.js"; +import { fileIdentityKey } from "../src/util/paths.js"; describe("Node modules resolution (opt-in) and path normalization", () => { it("treats packages as external by default; resolves to file with flag", async () => { @@ -103,6 +104,32 @@ describe("Node modules resolution (opt-in) and path normalization", () => { false, ); }); + + it("does not reuse a stale per-file module cache entry when resolveNodeModules turns on for a warm non-incremental build", async () => { + const root = await mkTmpDir("dg-nm-warm-toggle-"); + const nm = path.join(root, "node_modules", "my-pkg"); + const main = path.join(root, "main.js"); + await fsp.mkdir(nm, { recursive: true }); + await fsp.writeFile(main, 'import { thing } from "my-pkg";\nexport const used = thing;\n', "utf8"); + await fsp.writeFile(path.join(nm, "index.js"), "module.exports = 1;\n", "utf8"); + await fsp.writeFile(path.join(nm, "package.json"), JSON.stringify({ name: "my-pkg", main: "index.js" }), "utf8"); + + const cold = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const coldModule = cold.byFile.get(fileIdentityKey(main)); + const coldResolved = coldModule?.imports[0]?.resolved; + expect(typeof coldResolved === "string" && coldResolved.includes("node_modules")).toBe(false); + + const warm = await buildProjectIndex(root, { + cache: "disk", + graph: { resolveNodeModules: true }, + threads: 1, + }); + const warmModule = warm.byFile.get(fileIdentityKey(main)); + const warmResolved = warmModule?.imports[0]?.resolved; + expect( + typeof warmResolved === "string" && warmResolved.replace(/\\/g, "/").includes("node_modules/my-pkg/index.js"), + ).toBe(true); + }); it("normalizes paths to forward slashes in nodes and edges", async () => { const root = await mkTmpDir("dg-paths-"); const a = path.join(root, "a.ts"); diff --git a/tests/query-index.test.ts b/tests/query-index.test.ts index 27d2b3a4..3d34839d 100644 --- a/tests/query-index.test.ts +++ b/tests/query-index.test.ts @@ -785,6 +785,30 @@ describe("persistent query index", () => { store.close(); }); + it("gives each term its own prefetch budget so a rare term is not starved by a common early-path term", async () => { + const root = await createRepo(); + const databasePath = path.join(root, "query-candidate-fairness.sqlite"); + const store = new QueryIndexStore(databasePath); + const alphaFiles = Array.from({ length: 20 }, (_, index) => + preparedFile(`src/a${String(index).padStart(3, "0")}.ts`, ["const alphaNoise = 1;"]), + ); + const betaFile = preparedFile("src/zz-beta.ts", ["const betaValue = 1;"]); + const files = [...alphaFiles, betaFile]; + store.replaceFiles(files, [], { + ...expectedQueryIndexVersionMetadata(), + projectSnapshotIdentity: "snap-5", + projectRootIdentity: "root-5", + createdByCodegraphVersion: "test", + updatedAt: new Date().toISOString(), + }); + const paths = files.map((file) => file.path); + + const candidates = store.candidateChunksForTerms(["alpha", "beta"], paths, 10); + + expect(candidates.some((candidate) => candidate.path === "src/zz-beta.ts")).toBe(true); + store.close(); + }); + it("rejects absolute and traversing paths from persisted rows", async () => { const root = await createRepo(); expect(() => resolveQueryIndexSourcePath(root, "../outside.ts")).toThrow(/Invalid query index relative path/u); diff --git a/tests/sqlite-common.test.ts b/tests/sqlite-common.test.ts index 0e9f47fb..b66c29f2 100644 --- a/tests/sqlite-common.test.ts +++ b/tests/sqlite-common.test.ts @@ -68,6 +68,39 @@ describe("SQLite common helpers", () => { } }); + it("skips migrateTable when the schema is already at the current version", async () => { + const root = await mkTmpDir("dg-sqlite-versioned-current-"); + const db = new SqliteDatabase(path.join(root, "cache.sqlite")); + try { + let migrateCalls = 0; + const args = { + db, + tableName: "cache_entries", + schemaVersionKey: "cache_entries.schema_version", + schemaVersion: 1, + createTable: (target: SqliteDatabase) => { + target.exec( + "CREATE TABLE IF NOT EXISTS cache_entries (id TEXT PRIMARY KEY, payload TEXT NOT NULL DEFAULT '');", + ); + }, + migrateTable: () => { + migrateCalls += 1; + }, + }; + + ensureSqliteVersionedTableSchema(args); + expect(migrateCalls).toBe(1); + + ensureSqliteVersionedTableSchema(args); + ensureSqliteVersionedTableSchema(args); + + expect(migrateCalls).toBe(1); + expect(readSqliteSchemaVersion(db, "cache_entries.schema_version")).toEqual({ status: "ok", version: 1 }); + } finally { + db.close(); + } + }); + it("rebuilds versioned SQLite tables created by a newer schema", async () => { const root = await mkTmpDir("dg-sqlite-versioned-newer-"); const db = new SqliteDatabase(path.join(root, "cache.sqlite")); From ce073460771e3a8cd76033fc6ca30381577c746b Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 10:01:31 -0400 Subject: [PATCH 26/28] fix: prefer content-hash cacheSig over weak mtime:size sig in snapshot reuse - project-snapshot.ts: snapshotSignatureMatches now prefers the git- or content-hash-derived cacheSig over the cheap mtime:size sig whenever both sides have one. Non-strict, non-git disk caches previously reused a stale snapshot module or bloom filter after a same-size edit whose mtime got restored to its prior value, since sig alone can't distinguish that case. cacheSig is threaded end-to-end: ProjectIndexManifestEntry now carries an optional cacheSig, toProjectIndexManifestEntry synthesizes one from gitSig when the source lacks an explicit cacheSig (disk-manifest-derived entries), and the persisted SnapshotFileSignature schema + validator gained a matching optional field. - project-snapshot.ts: fixed a related pre-existing bug found while adding regression coverage - tryLoadProjectSnapshotModules looked up payload.fileSignatures by a fileIdentityKey-normalized key, but the payload's own keys were raw case-preserved paths, so the lookup always missed on case-insensitive filesystems (Windows/macOS), making snapshot module reuse silently non-functional there. Normalize the payload's keys before lookup, matching the pattern already used for persisted bloom filters. - project-snapshot.ts: isProjectIndexSnapshotPayload now validates the persisted languageExtensions field (must be undefined or a record of string values) before hydrating it, so a malformed cache entry is treated as a clean cache miss instead of crashing later in normalizeLanguageExtensions. - sqliteSchema.ts: ensureSqliteVersionedTableSchema now runs the idempotent createTable callback (not just skips migrateTable) when the schema is already at the current version, so a table that was dropped/partially restored while its version marker survived gets repaired instead of failing later index/prepare calls. Verification: tsc --noEmit clean, eslint/prettier clean, cache-invalidation, agent-session, cli-command-modules, query-index, sqlite-common, node-modules-and-paths, cache-modes, disk-cache-sqlite, cache-path-confinement, project-file-discovery, duplicates, and cli-startup-eager-modules suites all pass (830+ tests). --- src/indexer/build-cache/project-snapshot.ts | 40 +++++++++++--- src/indexer/build-index.ts | 11 +++- src/indexer/types.ts | 7 +++ src/util/sqliteSchema.ts | 5 ++ tests/cache-invalidation.test.ts | 58 +++++++++++++++++++-- 5 files changed, 108 insertions(+), 13 deletions(-) diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 20d2b442..18d1471e 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -104,10 +104,11 @@ type SerializedBloomFilter = { type SnapshotFileSignature = { sig: string; gitSig?: string; + cacheSig?: string; }; export type PersistedBloomFilters = { - get: (file: string, signature: Pick) => BloomFilter | undefined; + get: (file: string, signature: Pick) => BloomFilter | undefined; }; type SnapshotAnalysisReport = { @@ -491,7 +492,7 @@ export async function tryLoadProjectIndexSnapshot( export async function tryLoadProjectSnapshotModules( projectRoot: string, opts: BuildOptions | undefined, - fileSignatures: ReadonlyMap>, + fileSignatures: ReadonlyMap>, ): Promise | null> { if ((opts?.cache ?? "off") !== "disk") return null; try { @@ -511,12 +512,16 @@ export async function tryLoadProjectSnapshotModules( ) { return null; } + const normalizedFileSignatures = new Map( + Object.entries(payload.fileSignatures).map(([file, signature]) => [fileIdentityKey(file), signature]), + ); const modules = new Map(); for (const mod of payload.modules) { - const signature = fileSignatures.get(fileIdentityKey(mod.file)); - const snapshotSignature = payload.fileSignatures[fileIdentityKey(mod.file)]; + const moduleKey = fileIdentityKey(mod.file); + const signature = fileSignatures.get(moduleKey); + const snapshotSignature = normalizedFileSignatures.get(moduleKey); if (!signature || !snapshotSignature || !snapshotSignatureMatches(snapshotSignature, signature)) continue; - modules.set(fileIdentityKey(mod.file), mod); + modules.set(moduleKey, mod); } return modules; } catch { @@ -621,11 +626,19 @@ function createPersistedBloomFilters( function snapshotSignatureMatches( snapshotSignature: SnapshotFileSignature, - currentSignature: Pick, + currentSignature: Pick, ): boolean { const matchingGitSignature = !!snapshotSignature.gitSig && !!currentSignature.gitSig && snapshotSignature.gitSig === currentSignature.gitSig; - return matchingGitSignature || snapshotSignature.sig === currentSignature.sig; + if (matchingGitSignature) return true; + // `cacheSig` is git- or content-hash-derived (forced whenever caching is enabled without a git + // signature; see `fileSignature()`), so when both sides have it, it is a strictly stronger and + // authoritative identity check than the cheap `mtime:size` `sig`. Comparing bare `sig` alone + // would wrongly treat a same-size edit whose mtime got restored as unchanged. + if (snapshotSignature.cacheSig !== undefined && currentSignature.cacheSig !== undefined) { + return snapshotSignature.cacheSig === currentSignature.cacheSig; + } + return snapshotSignature.sig === currentSignature.sig; } export async function writeProjectIndexSnapshot( @@ -1124,6 +1137,7 @@ function isProjectIndexSnapshotPayload(value: unknown): value is ProjectIndexSna payload.modules.every(isModuleIndex) && isSnapshotFileSignatureRecord(payload.fileSignatures) && (payload.nativeMode === undefined || isSnapshotNativeMode(payload.nativeMode)) && + (payload.languageExtensions === undefined || isLanguageExtensionMap(payload.languageExtensions)) && (payload.bloomFilters === undefined || isSerializedBloomFilterRecord(payload.bloomFilters)) && (payload.analysis === undefined || isAnalysisSummary(payload.analysis)) && (payload.analysisReport === undefined || isSnapshotAnalysisReport(payload.analysisReport)) && @@ -1253,6 +1267,7 @@ function serializeSnapshotFileSignatures( serialized[cacheRelativePath(projectRoot, file)] = { sig: entry.sig, ...(entry.gitSig ? { gitSig: entry.gitSig } : {}), + ...(entry.cacheSig ? { cacheSig: entry.cacheSig } : {}), }; } return serialized; @@ -1266,7 +1281,11 @@ function isSnapshotFileSignatureRecord(value: unknown): value is Record; - return typeof signature.sig === "string" && (signature.gitSig === undefined || typeof signature.gitSig === "string"); + return ( + typeof signature.sig === "string" && + (signature.gitSig === undefined || typeof signature.gitSig === "string") && + (signature.cacheSig === undefined || typeof signature.cacheSig === "string") + ); } function deserializeBloomFilterCache( @@ -1293,6 +1312,11 @@ function isSerializedBloomFilterRecord(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + return Object.values(value).every((entry) => typeof entry === "string"); +} + function isSerializedBloomFilter(value: unknown): value is SerializedBloomFilter { if (!value || typeof value !== "object") return false; const filter = value as Partial; diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index c9f69429..2988ab60 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -479,15 +479,22 @@ function expandStarImports(modules: Map, opts?: BuildOption } } -function toProjectIndexManifestEntry(entry: Pick): ProjectIndexManifestEntry { +function toProjectIndexManifestEntry( + entry: Pick & { cacheSig?: string }, +): ProjectIndexManifestEntry { + // A git signature alone is already a strong content identity (`fileSignature()` derives + // `cacheSig` the same way: `gitSig ?? contentHash ?? sig`), so entries sourced from the disk + // manifest (which does not persist `cacheSig`) still get one whenever `gitSig` is available. + const cacheSig = entry.cacheSig ?? entry.gitSig; return { sig: entry.sig, ...(entry.gitSig ? { gitSig: entry.gitSig } : {}), + ...(cacheSig ? { cacheSig } : {}), }; } function projectIndexManifestEntries( - entries: Iterable]>, + entries: Iterable & { cacheSig?: string }]>, ): Map { return new Map(Array.from(entries, ([file, entry]) => [file, toProjectIndexManifestEntry(entry)])); } diff --git a/src/indexer/types.ts b/src/indexer/types.ts index 04dccb84..b2c87391 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -81,6 +81,13 @@ export type ResolvedExport = { kind: "resolved"; def: SymbolDef } | { kind: "nam export type ProjectIndexManifestEntry = { sig: string; gitSig?: string; + /** + * Content-identity signature (`FileSignature.cacheSig`), when it was available at signature + * computation time. Stronger than `sig` alone: unlike the cheap `mtime:size` fast path used + * when `cacheStrict` is off, this is git- or content-hash-derived and does not falsely match + * a changed file whose mtime and size happen to be restored to their prior values. + */ + cacheSig?: string; }; export type SqlNavigationCache = { diff --git a/src/util/sqliteSchema.ts b/src/util/sqliteSchema.ts index 3e4f6bf3..baa20c7a 100644 --- a/src/util/sqliteSchema.ts +++ b/src/util/sqliteSchema.ts @@ -86,6 +86,11 @@ export function ensureSqliteVersionedTableSchema(args: { // on-disk schema is actually behind; an already-current schema needs no work every // time the database is opened. args.migrateTable(args.db); + } else { + // Already current: skip the migration scan, but still guarantee the table exists via + // the idempotent `CREATE TABLE IF NOT EXISTS`, in case the version marker survived + // while the table itself was dropped or only partially restored. + args.createTable(args.db); } writeSqliteSchemaVersion(args.db, args.schemaVersionKey, args.schemaVersion); } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index d33c3097..c26a3580 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -269,6 +269,53 @@ describe("Cache invalidation and strict hashing", () => { expect(mod3.locals.some((l) => l.localName === "b")).toBe(true); }); + it("does not reuse a stale snapshot module or bloom filter when sig matches but cacheSig differs", async () => { + const root = await mkTmpDir("dg-snapshot-cachesig-"); + const utilPath = path.join(root, "util.ts"); + const v1 = `export function a(){ return 1 }\n`; + await fsp.writeFile(utilPath, v1, "utf8"); + + // Non-strict, non-git: `sig` alone is the cheap `mtime:size` form, so this scenario is + // exactly the one where the snapshot/bloom fast paths must fall back to the stronger + // content-hash-derived `cacheSig` rather than the weak `sig`. + const idx1 = await buildProjectIndex(root, { threads: 1, cache: "disk", cacheStrict: false }); + const utilFile = Array.from(idx1.byFile.keys()).find((f) => f.endsWith("/util.ts") || f.endsWith("\\util.ts"))!; + const persistedSignature = Array.from(idx1.manifestEntries ?? []).find( + ([file]) => fileIdentityKey(file) === utilFile, + )?.[1]; + if (!persistedSignature) throw new Error("Expected a persisted manifest entry for util.ts."); + + // A genuinely unchanged file (identical sig and cacheSig) must still be reused. + const unchangedSnapshotModules = await buildCache.tryLoadProjectSnapshotModules( + root, + { cache: "disk", cacheStrict: false }, + new Map([[fileIdentityKey(utilFile), persistedSignature]]), + ); + expect(unchangedSnapshotModules?.has(fileIdentityKey(utilFile))).toBe(true); + + // Simulate the reported collision directly (independent of filesystem mtime-write precision): + // the OS reports the exact same `mtime:size` `sig` string as before, but the real content + // (and therefore `cacheSig`) has changed. + const v2 = `export function b(){ return 2 }\n`; // same length as v1 + await fsp.writeFile(utilPath, v2, "utf8"); + const realCurrentSignature = await buildCache.fileSignature(utilFile, false, undefined, { forceContentHash: true }); + expect(realCurrentSignature.cacheSig).not.toBe(persistedSignature.cacheSig); + const collidingSignature = { ...realCurrentSignature, sig: persistedSignature.sig }; + + const snapshotModules = await buildCache.tryLoadProjectSnapshotModules( + root, + { cache: "disk", cacheStrict: false }, + new Map([[fileIdentityKey(utilFile), collidingSignature]]), + ); + expect(snapshotModules?.has(fileIdentityKey(utilFile))).toBe(false); + + const persistedBloomFilters = await buildCache.tryLoadPersistedBloomFilters(root, { + cache: "disk", + cacheStrict: false, + }); + expect(persistedBloomFilters?.get(utilFile, collidingSignature)).toBeUndefined(); + }); + it("rebuilds when the generated language-definition fingerprint changes without source changes", async () => { const root = await mkTmpDir("dg-implementation-fingerprint-"); const entryPath = path.join(root, "entry.ts"); @@ -1233,9 +1280,14 @@ describe("Cache invalidation and strict hashing", () => { expect(await fsp.readFile(manifestPathFor(root), "utf8")).toBe(manifestBefore); const moduleIndex = incremental.byFile.get(fileIdentityKey(normalize(filePath))); expect(moduleIndex?.locals.some((local) => local.localName === "snap")).toBe(true); - expect([...(incremental.manifestEntries?.entries() ?? [])]).toEqual([ - ...(initial.manifestEntries?.entries() ?? []), - ]); + // Compare sig/gitSig identity only: `cacheSig` is an optional strengthening field whose + // presence depends on which internal reuse path populated the entry (fresh computation + // vs. disk-manifest-derived reuse), not a guarantee both builds must expose identically. + const toIdentity = (entries: [string, { sig: string; gitSig?: string }][]) => + entries.map(([file, entry]) => [file, { sig: entry.sig, gitSig: entry.gitSig }]); + expect(toIdentity([...(incremental.manifestEntries?.entries() ?? [])])).toEqual( + toIdentity([...(initial.manifestEntries?.entries() ?? [])]), + ); } finally { prepSpy.mockRestore(); signatureSpy.mockRestore(); From 5d3e50e21834135c5a421f7fb8eacaf380738e13 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 14:30:22 -0400 Subject: [PATCH 27/28] fix: address fifth batch of review findings on cache identity and node-module toggle - build-index.ts: fold resolveNodeModules into the module-cache signature (moduleCacheSignatureForFile) instead of only gating reads for one direction of the toggle. Previously a build with resolveNodeModules enabled would overwrite the same cache row (keyed only by content signature) with resolved node_modules import targets; a later default build (resolveNodeModules off) could then reuse that row and leak resolved paths where the contract expects external packages. Both directions now key to distinct rows. - build-index.ts: incremental builds finalize manifestEntries from ManifestFileEntry (graph-edge derived), which never carries cacheSig. Overlay each entry with the real FileSignature.cacheSig this build already computed (content-hash-derived for non-git files, since caching is enabled whenever this path runs) before handing manifestEntries to finalizeProjectIndex, so incremental writes preserve the same strong identity a cold build produces instead of leaving persisted snapshot/ bloom signatures to silently fall back to the weak mtime:size sig. - location.ts: resolveCacheAnchor now validates cacheLocation against the same project/repo/user/absolute-path contract the config schema already enforces, throwing an actionable error instead of silently resolving a typo'd relative string against the process working directory. Verification: tsc --noEmit clean, eslint/prettier clean, cache-invalidation, node-modules-and-paths, agent-session, query-index, cache-modes, disk-cache-sqlite, cache-path-confinement, project-file-discovery, duplicates, sqlite-common, cli-command-modules, and cli-startup-eager-modules suites all pass (418+ tests in this sweep). Each new regression test independently confirmed to fail without its fix. --- src/indexer/build-cache/location.ts | 7 ++++++- src/indexer/build-index.ts | 24 ++++++++++++++++++++---- tests/cache-invalidation.test.ts | 27 +++++++++++++++++++++++++++ tests/node-modules-and-paths.test.ts | 25 +++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/indexer/build-cache/location.ts b/src/indexer/build-cache/location.ts index 60e1b634..64764fb3 100644 --- a/src/indexer/build-cache/location.ts +++ b/src/indexer/build-cache/location.ts @@ -70,7 +70,12 @@ export function resolveCacheAnchor(projectRoot: string, opts?: BuildOptions): Ca const location = opts?.cacheLocation; if (location === "project") return { anchor: path.resolve(projectRoot), layer: "project" }; if (location === "user") return { anchor: resolveCodegraphUserCacheRoot(), layer: "user" }; - if (location && location !== "repo") return { anchor: path.resolve(location), layer: "explicit" }; + if (location && location !== "repo") { + if (!path.isAbsolute(location)) { + throw new Error(`Cache location must be "project", "repo", "user", or an absolute path. Received: "${location}"`); + } + return { anchor: path.resolve(location), layer: "explicit" }; + } return findRepositoryAnchor(projectRoot); } diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 2988ab60..1387ec34 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -403,15 +403,20 @@ function graphEdgeKey(edge: Edge): string { async function moduleCacheSignatureForFile(file: string, sigInfo: FileSignature, opts?: BuildOptions): Promise { const baseSignature = await cacheSignatureForFile(file, sigInfo, opts); const normalizedExtensions = normalizeLanguageExtensions(opts?.languageExtensions); - if (!normalizedExtensions) return baseSignature; + const resolveNodeModules = normalizeGraphOptions(opts?.graph).resolveNodeModules; + if (!normalizedExtensions && !resolveNodeModules) return baseSignature; // Combine via a hash rather than raw concatenation: the disk cache stores this string in a // SQLite TEXT column, and node:sqlite's DatabaseSync silently truncates TEXT bind parameters // at embedded NUL bytes, so a raw separator character risks the stored and freshly-computed // signatures never matching (permanent cache miss) if either baseSignature or the serialized - // extensions ever contained one. + // extensions ever contained one. A cached ModuleIndex's ImportBinding.resolved values differ + // depending on whether resolveNodeModules was on at write time (resolved node_modules targets + // vs. external), so that state must be part of the key too, not just gate reuse for one + // direction of the toggle. const hash = crypto.createHash("sha1"); hash.update(baseSignature); - hash.update(JSON.stringify(Object.entries(normalizedExtensions))); + hash.update(JSON.stringify(Object.entries(normalizedExtensions ?? {}))); + hash.update(resolveNodeModules ? "\0resolveNodeModules" : ""); return hash.digest("hex"); } @@ -1705,7 +1710,18 @@ export async function buildProjectIndexIncremental( modules, parsedMap, bloomFilterCache, - manifestEntries: projectIndexManifestEntries(manifestEntries), + manifestEntries: projectIndexManifestEntries( + // `manifestEntries` (a `ManifestFileEntry`) never carries `cacheSig` -- that field + // only lives on `FileSignature`. Overlay each entry with the `cacheSig` this build + // actually computed (content-hash-derived for non-git files, since caching is enabled + // whenever this path runs) so incremental writes preserve the same strong identity a + // cold build produces, instead of leaving snapshot/bloom reuse to fall back to the + // weak `mtime:size` `sig`. + Array.from(manifestEntries, ([file, entry]) => { + const cacheSig = fileSignatures.get(file)?.cacheSig; + return [file, { ...entry, ...(cacheSig ? { cacheSig } : {}) }] as const; + }), + ), buildReport: report, }); await writeProjectIndexSnapshot( diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index c26a3580..b8167599 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -316,6 +316,25 @@ describe("Cache invalidation and strict hashing", () => { expect(persistedBloomFilters?.get(utilFile, collidingSignature)).toBeUndefined(); }); + it("preserves content-hash cacheSig for changed files' manifestEntries after an incremental build", async () => { + const root = await mkTmpDir("dg-incremental-cachesig-"); + const filePath = path.join(root, "entry.ts"); + await fsp.writeFile(filePath, "export const value = 1;\n", "utf8"); + await buildProjectIndex(root, { cache: "disk", cacheStrict: false, threads: 1 }); + + // Non-git: a content change here is the exact scenario snapshotSignatureMatches relies on + // cacheSig to distinguish from a same-mtime/size collision. Prove the incremental write path + // actually persists that stronger identity instead of leaving it undefined. + await fsp.writeFile(filePath, "export const value = 2;\n", "utf8"); + const incremental = await buildProjectIndexIncremental(root, { cache: "disk", cacheStrict: false, threads: 1 }); + + const entry = Array.from(incremental.manifestEntries ?? []).find( + ([file]) => fileIdentityKey(file) === fileIdentityKey(filePath), + )?.[1]; + expect(entry?.cacheSig).toBeDefined(); + expect(entry?.cacheSig).toMatch(/^[a-f0-9]{40}$/); + }); + it("rebuilds when the generated language-definition fingerprint changes without source changes", async () => { const root = await mkTmpDir("dg-implementation-fingerprint-"); const entryPath = path.join(root, "entry.ts"); @@ -2645,6 +2664,14 @@ describe("Cache invalidation and strict hashing", () => { } }); + it("rejects a cacheLocation that is not project/repo/user/absolute", async () => { + const root = await mkTmpDir("dg-cache-invalid-location-"); + await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); + expect(() => buildCache.resolveCacheLocation(root, { cacheLocation: "relative-dir" })).toThrow( + /Cache location must be "project", "repo", "user", or an absolute path/, + ); + }); + it("reuses a legacy v4 project snapshot missing fileSignatures instead of crashing to a forced miss", async () => { const root = await mkTmpDir("dg-cache-legacy-v4-"); await fsp.writeFile(path.join(root, "entry.ts"), "export const entry = 1;\n", "utf8"); diff --git a/tests/node-modules-and-paths.test.ts b/tests/node-modules-and-paths.test.ts index 7e2dfba8..74f831de 100644 --- a/tests/node-modules-and-paths.test.ts +++ b/tests/node-modules-and-paths.test.ts @@ -130,6 +130,31 @@ describe("Node modules resolution (opt-in) and path normalization", () => { typeof warmResolved === "string" && warmResolved.replace(/\\/g, "/").includes("node_modules/my-pkg/index.js"), ).toBe(true); }); + it("does not reuse a stale per-file module cache entry when resolveNodeModules turns off for a warm non-incremental build", async () => { + const root = await mkTmpDir("dg-nm-warm-toggle-off-"); + const nm = path.join(root, "node_modules", "my-pkg"); + const main = path.join(root, "main.js"); + await fsp.mkdir(nm, { recursive: true }); + await fsp.writeFile(main, 'import { thing } from "my-pkg";\nexport const used = thing;\n', "utf8"); + await fsp.writeFile(path.join(nm, "index.js"), "module.exports = 1;\n", "utf8"); + await fsp.writeFile(path.join(nm, "package.json"), JSON.stringify({ name: "my-pkg", main: "index.js" }), "utf8"); + + const warm = await buildProjectIndex(root, { + cache: "disk", + graph: { resolveNodeModules: true }, + threads: 1, + }); + const warmModule = warm.byFile.get(fileIdentityKey(main)); + const warmResolved = warmModule?.imports[0]?.resolved; + expect( + typeof warmResolved === "string" && warmResolved.replace(/\\/g, "/").includes("node_modules/my-pkg/index.js"), + ).toBe(true); + + const cold = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const coldModule = cold.byFile.get(fileIdentityKey(main)); + const coldResolved = coldModule?.imports[0]?.resolved; + expect(typeof coldResolved === "string" && coldResolved.includes("node_modules")).toBe(false); + }); it("normalizes paths to forward slashes in nodes and edges", async () => { const root = await mkTmpDir("dg-paths-"); const a = path.join(root, "a.ts"); From adb3ec65a8d3a436e762b82234d0f3270b9ef8e1 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 16:36:37 -0400 Subject: [PATCH 28/28] fix: address sixth batch of review findings on portable cache identity - manifest.ts: loadManifest() now updates the migrated manifest's projectRoot to the active root after rebasing entries/edges/ transientFiles/symlinkDirectories. Previously it retained the stale pre-move root, so build-index.ts's cachedFileEdgesProjectRoot check (compared against it in graph-edge-collector.ts's collectEdgesForFile) rejected every cached edge after a project move, forcing a full graph reparse on the very next rebuild despite the manifest entries themselves being valid and correctly rebased. - unitCache.ts: duplicateUnitCacheSignature() now prefers the git- or content-hash-derived cacheSig over the weak mtime:size sig, matching the same fix already applied to snapshot/bloom signature matching. Without it, a non-Git project's duplicate-unit disk/memory cache could reuse stale duplicate results after a same-size edit whose mtime got restored. - project-snapshot.ts: tryLoadProjectSnapshotModules() now also normalizes the caller-supplied fileSignatures map by fileIdentityKey before lookup, not just the persisted payload's own fileSignatures record. The caller (prepareFileSignatures in build-index.ts) keys its map by whatever raw discovered display path each file was found under, which can differ in case from the fileIdentityKey-normalized module key on case-insensitive filesystems (Windows/macOS), silently defeating snapshot module reuse. Verification: tsc --noEmit clean, eslint/prettier clean, cache-invalidation, duplicates, node-modules-and-paths, agent-session, query-index, cache-modes, disk-cache-sqlite, cache-path-confinement, project-file-discovery, sqlite-common, cli-command-modules, and cli-startup-eager-modules suites all pass (421+ tests). Each new regression test independently confirmed to fail without its corresponding fix. --- src/duplicates/unitCache.ts | 5 ++- src/indexer/build-cache/manifest.ts | 5 +++ src/indexer/build-cache/project-snapshot.ts | 8 +++- tests/cache-invalidation.test.ts | 42 +++++++++++++++++++++ tests/duplicates.test.ts | 24 ++++++++++++ 5 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index 8ff82dbe..fd2f651a 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -165,7 +165,10 @@ export function duplicateUnitCacheSignature( const root = projectRoot ?? index.projectRoot; const entry = index.manifestEntries?.get(file) ?? (root ? index.manifestEntries?.get(cacheRelativePath(root, file)) : undefined); - return entry?.gitSig ?? entry?.sig; + // `cacheSig` is git- or content-hash-derived and distinguishes a same-size edit whose mtime + // got restored; falling back straight to `sig` would let a non-Git project reuse stale + // duplicate units for a file whose content actually changed. + return entry?.cacheSig ?? entry?.gitSig ?? entry?.sig; } export function duplicateUnitCacheKey(file: string, variant: string): string { diff --git a/src/indexer/build-cache/manifest.ts b/src/indexer/build-cache/manifest.ts index 6bd8b1bc..679130ab 100644 --- a/src/indexer/build-cache/manifest.ts +++ b/src/indexer/build-cache/manifest.ts @@ -314,6 +314,11 @@ export async function loadManifest(projectRoot: string, opts?: BuildOptions): Pr const migrated: IndexManifest = { ...parsed, version: MANIFEST_VERSION, + // Entries, edges, transientFiles, and symlinkDirectories above are already rebased to the + // active `projectRoot`. Update the stored provenance to match, or `cachedFileEdgesProjectRoot` + // (build-index.ts) keeps pointing at the pre-move root and `collectEdgesForFile` + // (graph-edge-collector.ts) rejects every cached edge, forcing a full reparse after a move. + projectRoot: path.resolve(projectRoot).replace(/\\/g, "/"), files: transformManifestEntries(projectRoot, relativeFiles, false), transientFiles: sanitizeManifestTransientFilesForRoot(projectRoot, parsed.projectRoot, parsed.transientFiles), ...(symlinkDirectories !== undefined ? { symlinkDirectories } : {}), diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 18d1471e..3c0f5c28 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -515,10 +515,16 @@ export async function tryLoadProjectSnapshotModules( const normalizedFileSignatures = new Map( Object.entries(payload.fileSignatures).map(([file, signature]) => [fileIdentityKey(file), signature]), ); + // `fileSignatures` (caller-supplied) is keyed by whatever discovered display path each file + // was found under, not necessarily `fileIdentityKey`-normalized; on a case-insensitive + // filesystem an uppercase path segment would otherwise miss this lookup. + const normalizedCurrentSignatures = new Map( + Array.from(fileSignatures, ([file, signature]) => [fileIdentityKey(file), signature]), + ); const modules = new Map(); for (const mod of payload.modules) { const moduleKey = fileIdentityKey(mod.file); - const signature = fileSignatures.get(moduleKey); + const signature = normalizedCurrentSignatures.get(moduleKey); const snapshotSignature = normalizedFileSignatures.get(moduleKey); if (!signature || !snapshotSignature || !snapshotSignatureMatches(snapshotSignature, signature)) continue; modules.set(moduleKey, mod); diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index b8167599..e9fa8983 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -316,6 +316,27 @@ describe("Cache invalidation and strict hashing", () => { expect(persistedBloomFilters?.get(utilFile, collidingSignature)).toBeUndefined(); }); + it("resolves snapshot module reuse when the caller's fileSignatures map is keyed by a raw display path", async () => { + const root = await mkTmpDir("dg-snapshot-display-path-"); + const utilPath = path.join(root, "util.ts").replace(/\\/g, "/"); + await fsp.writeFile(utilPath, "export function a(){ return 1 }\n", "utf8"); + + const idx1 = await buildProjectIndex(root, { threads: 1, cache: "disk", cacheStrict: false }); + const utilFile = Array.from(idx1.byFile.keys()).find((f) => f.endsWith("/util.ts") || f.endsWith("\\util.ts"))!; + const currentSignature = await buildCache.fileSignature(utilPath, false, undefined, { forceContentHash: true }); + + // `prepareFileSignatures` (build-index.ts) keys its map by whatever raw display path each + // file was discovered under, not `fileIdentityKey`. On a case-insensitive filesystem a + // mixed-case root (as `mkTmpDir` produces on Windows) makes that key differ from the + // lowercase `fileIdentityKey` form the snapshot module lookup uses internally. + const snapshotModules = await buildCache.tryLoadProjectSnapshotModules( + root, + { cache: "disk", cacheStrict: false }, + new Map([[utilPath, currentSignature]]), + ); + expect(snapshotModules?.has(fileIdentityKey(utilFile))).toBe(true); + }); + it("preserves content-hash cacheSig for changed files' manifestEntries after an incremental build", async () => { const root = await mkTmpDir("dg-incremental-cachesig-"); const filePath = path.join(root, "entry.ts"); @@ -2570,6 +2591,27 @@ describe("Cache invalidation and strict hashing", () => { expect(report.files?.cached).toBeGreaterThan(0); }); + it("reuses cached graph edges (not just modules) after moving a project tree", async () => { + const sourceRoot = await mkTmpDir("dg-cache-move-edges-source-"); + const movedRoot = `${sourceRoot}-moved`; + await fsp.writeFile(path.join(sourceRoot, "dependency.ts"), "export const dependency = 1;\n", "utf8"); + await fsp.writeFile(path.join(sourceRoot, "entry.ts"), "export { dependency } from './dependency';\n", "utf8"); + await buildProjectIndex(sourceRoot, { cache: "disk", threads: 1 }); + await fsp.rename(sourceRoot, movedRoot); + + // A stale `manifest.projectRoot` (left pointing at the pre-move root after rebasing entries) + // makes `collectEdgesForFile`'s `cachedFileEdgesProjectRoot` check reject every cached edge, + // forcing every unchanged file back through source parsing on the very next rebuild. + const prepSpy = vi.spyOn(filePrep, "prepareSourceInput"); + try { + const moved = await buildProjectIndex(movedRoot, { cache: "disk", threads: 1 }); + expect(moved.byFile.has(fileIdentityKey(normalize(path.join(movedRoot, "entry.ts"))))).toBe(true); + expect(prepSpy).not.toHaveBeenCalled(); + } finally { + prepSpy.mockRestore(); + } + }); + it("reuses symlink directory hints after moving a project tree", async () => { const sourceRoot = await mkTmpDir("dg-cache-move-symlink-source-"); const movedRoot = `${sourceRoot}-moved`; diff --git a/tests/duplicates.test.ts b/tests/duplicates.test.ts index a0930d92..c6425176 100644 --- a/tests/duplicates.test.ts +++ b/tests/duplicates.test.ts @@ -15,6 +15,7 @@ import { getDuplicateAstContext } from "../src/duplicates/units.js"; import { DUPLICATE_UNIT_CACHE_VERSION, closeDuplicateUnitCacheForIndex, + duplicateUnitCacheSignature, duplicateUnitCacheVariant, duplicateUnitDiskCache, tryLoadDuplicateUnitsFromCache, @@ -2539,3 +2540,26 @@ export function processInvoiceItems(items: Array<{ price: number; qty: number }> } } }); + +test("C5: duplicate unit cache signature prefers the content-hash cacheSig over the weak sig", async () => { + const root = await makeTempProject(); + let index; + try { + const source = "export function unitCacheSigProbe(value: number): number {\n return value + 1;\n}\n"; + const file = await writeProjectFile(root, "src/probe.ts", source); + index = await buildProjectIndex(root, { cache: "disk" }); + const entry = index.manifestEntries?.get(file); + if (!entry) throw new Error("expected a manifest entry for probe.ts"); + + // Non-Git project: `cacheSig` is content-hash-derived while `sig` is the cheap + // `mtime:size` form, so they are never equal here. Preferring `cacheSig` is what + // prevents a same-size edit whose mtime got restored from reusing stale duplicate units. + expect(entry.cacheSig).toBeDefined(); + expect(entry.cacheSig).not.toBe(entry.sig); + expect(duplicateUnitCacheSignature(index, file)).toBe(entry.cacheSig); + } finally { + if (index) { + closeDuplicateUnitCacheForIndex(index); + } + } +});