diff --git a/src/chunking/chunkFile.ts b/src/chunking/chunkFile.ts index f845e90a..704cf520 100644 --- a/src/chunking/chunkFile.ts +++ b/src/chunking/chunkFile.ts @@ -39,7 +39,12 @@ export interface ChunkFileOptions { * @param opts Chunking options * @returns Array of semantic chunks */ -export function chunkFile(opts: ChunkFileOptions): Chunk[] { +export type ChunkFileWithSymbolsResult = { + chunks: Chunk[]; + symbolChunks: Chunk[]; +}; + +export function chunkFileWithSymbols(opts: ChunkFileOptions): ChunkFileWithSymbolsResult { const { language, source, filePath, minTokens = 150, maxTokens = 400, tokenizer = countWhitespaceTokens } = opts; const matches = getChunkMatches(language, source, filePath); const newlineOffsets = collectNewlineOffsets(source); @@ -79,7 +84,43 @@ export function chunkFile(opts: ChunkFileOptions): Chunk[] { } preliminaryChunks.sort((left, right) => left.sourceStart - right.sourceStart || right.sourceEnd - left.sourceEnd); - const mergedChunks = mergeSmallChunks(preliminaryChunks, minTokens, maxTokens, tokenizer); + + if (minTokens <= 1) { + const completeChunks = fillGapsWithMiscChunks( + preliminaryChunks, + source, + language.id, + filePath, + tokenizer, + 1, + maxTokens, + newlineOffsets, + ); + const chunks = withStableChunkIds( + completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), + language.id, + filePath, + ); + return { chunks, symbolChunks: chunks }; + } + + const completeSymbolChunks = fillGapsWithMiscChunks( + preliminaryChunks, + source, + language.id, + filePath, + tokenizer, + 1, + maxTokens, + newlineOffsets, + ); + const symbolChunks = withStableChunkIds( + completeSymbolChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), + language.id, + filePath, + ); + + const mergedChunks = mergeSmallChunks([...preliminaryChunks], minTokens, maxTokens, tokenizer); const completeChunks = fillGapsWithMiscChunks( mergedChunks, source, @@ -90,12 +131,17 @@ export function chunkFile(opts: ChunkFileOptions): Chunk[] { maxTokens, newlineOffsets, ); - - return withStableChunkIds( + const chunks = withStableChunkIds( completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), language.id, filePath, ); + + return { chunks, symbolChunks }; +} + +export function chunkFile(opts: ChunkFileOptions): Chunk[] { + return chunkFileWithSymbols(opts).chunks; } function appendBlockChunks( diff --git a/src/duplicates/unitCache.ts b/src/duplicates/unitCache.ts index fb3f803e..fd3399cf 100644 --- a/src/duplicates/unitCache.ts +++ b/src/duplicates/unitCache.ts @@ -360,43 +360,80 @@ export function tryLoadDuplicateUnitsFromCache( } } -export function writeDuplicateUnitsToCache( +export type PendingDuplicateUnitCacheWrite = { + file: string; + variant: string; + units: DuplicateInternalUnit[]; +}; + +export function writeDuplicateUnitsBatchToCache( index: ProjectIndex, - file: string, - variant: string, - units: DuplicateInternalUnit[], + writes: readonly PendingDuplicateUnitCacheWrite[], ): void { - const sig = duplicateUnitCacheSignature(index, file); - if (!sig) return; - const key = duplicateUnitCacheKey(file, variant); + if (!writes.length) return; + const root = index.projectRoot ?? ""; if (index.cacheMode === "memory") { - writeDuplicateUnitMemoryCache(key, { sig, units }); + for (const write of writes) { + const sig = duplicateUnitCacheSignature(index, write.file); + if (!sig) continue; + writeDuplicateUnitMemoryCache(duplicateUnitCacheKey(write.file, write.variant), { + sig, + units: write.units, + }); + } return; } - if (index.cacheMode === "disk") { - try { - const entry = duplicateUnitDiskCache(index); - const root = index.projectRoot ?? ""; + if (index.cacheMode !== "disk") return; + try { + const entry = duplicateUnitDiskCache(index); + const preparedWrites: Array<{ + file: string; + variant: string; + sig: string; + payload: Buffer; + }> = []; + for (const write of writes) { + const sig = duplicateUnitCacheSignature(index, write.file); + if (!sig) continue; const payload = brotliCompressSync( - JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(units), true)), - { - params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, - }, + JSON.stringify(transformDuplicateUnits(root, serializeDuplicateUnits(write.units), true)), + { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }, ); - entry?.statements?.write.run( - cacheRelativePath(root, file), - variant, + preparedWrites.push({ + file: cacheRelativePath(root, write.file), + variant: write.variant, sig, - DUPLICATE_UNIT_CACHE_VERSION, payload, - Date.now(), - ); - } catch { - // best-effort cache + }); } + if (!preparedWrites.length || !entry?.db || !entry.statements) return; + const now = Date.now(); + entry.db.transaction(() => { + for (const write of preparedWrites) { + entry.statements?.write.run( + write.file, + write.variant, + write.sig, + DUPLICATE_UNIT_CACHE_VERSION, + write.payload, + now, + ); + } + })(); + } catch { + // best-effort cache } } +export function writeDuplicateUnitsToCache( + index: ProjectIndex, + file: string, + variant: string, + units: DuplicateInternalUnit[], +): void { + writeDuplicateUnitsBatchToCache(index, [{ file, variant, units }]); +} + export function deserializeDuplicateUnits(value: unknown): DuplicateInternalUnit[] | null { if (!Array.isArray(value) || !value.every(isDuplicateSerializedUnit)) return null; return value.map((unit) => ({ @@ -430,7 +467,7 @@ function transformDuplicateUnits( ): DuplicateSerializedUnit[] { return units.map((unit) => ({ ...unit, - file: transformDuplicatePath(root, unit.file, toRelative), + file: cacheRelativePath(root, unit.file), absoluteFile: transformDuplicatePath(root, unit.absoluteFile, toRelative), handle: transformDuplicatePath(root, unit.handle, toRelative), fileHandle: transformDuplicatePath(root, unit.fileHandle, toRelative), diff --git a/src/duplicates/units.ts b/src/duplicates/units.ts index 58af155b..95793731 100644 --- a/src/duplicates/units.ts +++ b/src/duplicates/units.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import fsp from "node:fs/promises"; import path from "node:path"; import { LANG_CONFIGS } from "../bootstrap/treeSitterLanguages.js"; -import { chunkFile, type Chunk } from "../chunking/chunkFile.js"; +import { chunkFile, chunkFileWithSymbols, type Chunk } from "../chunking/chunkFile.js"; import { chunkTextFile } from "../chunking/chunkTextFile.js"; import { countDuplicateTokens, @@ -25,7 +25,13 @@ import { maskJsLikeCommentsStringsAndRegex } from "../util/comments.js"; import { collectLineStartOffsets } from "../util/lines.js"; import { assertFilePathWithinRoot, fileIdentityKey, normalizePath, toProjectDisplayPath } from "../util/paths.js"; import { logWithLevel } from "../logging.js"; -import { duplicateUnitCacheVariant, tryLoadDuplicateUnitsFromCache, writeDuplicateUnitsToCache } from "./unitCache.js"; +import { + duplicateUnitCacheVariant, + tryLoadDuplicateUnitsFromCache, + writeDuplicateUnitsBatchToCache, + writeDuplicateUnitsToCache, + type PendingDuplicateUnitCacheWrite, +} from "./unitCache.js"; import type { CollectedDuplicateUnits, DuplicateAstContext, @@ -290,6 +296,29 @@ export function makeDuplicateChunks( return chunkTextFile({ source, filePath, languageId, minTokens, maxTokens, tokenizer: countDuplicateTokens }); } +export function makeDuplicateChunksWithSymbols( + filePath: string, + languageId: string, + textOnly: boolean, + source: string, + minTokens: number, + maxTokens: number, +): { chunks: Chunk[]; symbolChunks: Chunk[] } { + const langConfig = LANG_CONFIGS[chunkLanguageAliases[languageId] ?? languageId]; + if (langConfig && !textOnly) { + return chunkFileWithSymbols({ + language: langConfig, + source, + filePath, + minTokens, + maxTokens, + tokenizer: countDuplicateTokens, + }); + } + const chunks = chunkTextFile({ source, filePath, languageId, minTokens, maxTokens, tokenizer: countDuplicateTokens }); + return { chunks, symbolChunks: [] }; +} + export function makeSymbolSourceChunks( filePath: string, languageId: string, @@ -506,6 +535,7 @@ export async function collectDuplicateUnits( let belowThresholdUnits = 0; const belowThresholdUnitsByFile = new Map(); + const pendingWrites: PendingDuplicateUnitCacheWrite[] = []; for (const file of normalizedFiles) { const cachedUnits = tryLoadDuplicateUnitsFromCache(index, file, variant); const fileUnits = @@ -521,7 +551,7 @@ export async function collectDuplicateUnits( astContextCache, )); if (!cachedUnits) { - writeDuplicateUnitsToCache(index, file, variant, fileUnits); + pendingWrites.push({ file, variant, units: fileUnits }); } for (const unit of fileUnits) { if (!shouldKeepUnit(unit, options.includeSmall, options.minTokens)) { @@ -532,6 +562,9 @@ export async function collectDuplicateUnits( units.push(unit); } } + if (pendingWrites.length) { + writeDuplicateUnitsBatchToCache(index, pendingWrites); + } units.sort((left, right) => { const fileCompare = left.absoluteFile.localeCompare(right.absoluteFile); @@ -567,8 +600,14 @@ export async function buildDuplicateUnitsForFile( } const astContext = language.textOnly ? undefined : await getDuplicateAstContext(index, file, source, astContextCache); - const chunks = makeDuplicateChunks(file, language.id, language.textOnly, source, minTokens, maxTokens); - const symbolChunks = makeSymbolSourceChunks(file, language.id, language.textOnly, source, maxTokens); + const { chunks, symbolChunks } = makeDuplicateChunksWithSymbols( + file, + language.id, + language.textOnly, + source, + minTokens, + maxTokens, + ); const symbolUnits = (moduleIndex?.locals ?? []) .map((symbol) => { const chunk = findChunkForSymbol(symbol, symbolChunks); diff --git a/src/indexer/build-cache.ts b/src/indexer/build-cache.ts index 9eb08dd2..412bb87d 100644 --- a/src/indexer/build-cache.ts +++ b/src/indexer/build-cache.ts @@ -21,14 +21,19 @@ export { fileSignature, pruneDiskModuleCache, tryLoadFromCache, + writeModulesToCache, writeToCache, type FileSignature, + type PendingModuleCacheWrite, } from "./build-cache/module-cache.js"; export { + BLOOM_FILTER_SNAPSHOT_FILENAME, + BLOOM_FILTER_SNAPSHOT_VERSION, createProjectSnapshotIdentity, projectSnapshotFilesSignature, tryLoadDetailedSymbolGraphSnapshot, tryLoadPersistedBloomFilters, + tryLoadProjectSnapshotModules, tryLoadProjectIndexSnapshot, writeDetailedSymbolGraphSnapshot, writeProjectIndexSnapshot, diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index 5f2a3778..94bb081c 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -107,7 +107,9 @@ function isForbiddenAnchor(candidate: string): boolean { const resolved = path.resolve(candidate); const home = path.resolve(os.homedir()); const parsed = path.parse(resolved); - return fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root); + return ( + fileIdentityKey(resolved) === fileIdentityKey(home) || fileIdentityKey(resolved) === fileIdentityKey(parsed.root) + ); } function findRepositoryAnchor(projectRoot: string): CacheAnchorResolution { @@ -168,9 +170,7 @@ export function cacheRoot(projectRoot: string, opts?: BuildOptions): string { if (path.basename(configured) === namespace) return configured; return path.join(configured, namespace); } - const base = opts?.cacheLocation === "user" - ? resolveCodegraphUserCacheRoot() - : path.join(anchor, ".codegraph-cache"); + const base = opts?.cacheLocation === "user" ? resolveCodegraphUserCacheRoot() : path.join(anchor, ".codegraph-cache"); const candidate = path.join(path.resolve(base), "index-v1", namespace); if (!sameRoot && !opts?.cacheLocation) { const legacy = path.join(root, ".codegraph-cache", "index-v1"); @@ -238,7 +238,7 @@ function ensureModuleCacheSchema(db: SqliteDatabase, projectRoot: string): void db.exec("CREATE INDEX IF NOT EXISTS idx_module_cache_sig ON module_cache(sig);"); } -function getDiskModuleCache(projectRoot: string, opts?: BuildOptions): DiskModuleCache { +export function getDiskModuleCache(projectRoot: string, opts?: BuildOptions): DiskModuleCache { const dbPath = diskCacheDatabasePath(projectRoot, opts); const existing = diskModuleCaches.get(dbPath); if (existing) return existing; @@ -433,7 +433,8 @@ function isModuleIndex(value: unknown): value is ModuleIndex { function transformModulePaths(projectRoot: string, module: ModuleIndex, toRelative: boolean): ModuleIndex { const copy = structuredClone(module); - const transform = (file: string): string => (toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file)); + const transform = (file: string): string => + toRelative ? cacheRelativePath(projectRoot, file) : cacheAbsolutePath(projectRoot, file); copy.file = transform(copy.file); for (const local of copy.locals) local.file = transform(local.file); for (const entry of copy.exports) { @@ -493,28 +494,48 @@ export function tryLoadFromCache( return null; } -export function writeToCache( +export type PendingModuleCacheWrite = { + file: string; + sig: string; + mod: ModuleIndex; +}; + +export function writeModulesToCache( projectRoot: string, - file: string, - sig: string, - mod: ModuleIndex, + writes: readonly PendingModuleCacheWrite[], opts?: BuildOptions, ): void { + if (!writes.length) return; const mode = opts?.cache ?? "off"; if (mode === "memory") { - lruMapSet( - memoryCache, - memoryCacheKey(projectRoot, file), - { version: PARSED_CACHE_VERSION, sig, mod }, - MAX_MEMORY_CACHE_ENTRIES, - ); + for (const write of writes) { + lruMapSet( + memoryCache, + memoryCacheKey(projectRoot, write.file), + { version: PARSED_CACHE_VERSION, sig: write.sig, mod: write.mod }, + MAX_MEMORY_CACHE_ENTRIES, + ); + } } else if (mode === "disk") { try { const cache = getDiskModuleCache(projectRoot, opts); - const payload = brotliCompressSync(JSON.stringify(transformModulePaths(projectRoot, mod, true)), { - params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, - }); - cache.write.run(cacheRelativePath(projectRoot, file), sig, PARSED_CACHE_VERSION, payload, Date.now()); + const now = Date.now(); + const preparedWrites: Array<{ file: string; sig: string; payload: Buffer }> = []; + for (const write of writes) { + const payload = brotliCompressSync(JSON.stringify(transformModulePaths(projectRoot, write.mod, true)), { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }); + preparedWrites.push({ + file: cacheRelativePath(projectRoot, write.file), + sig: write.sig, + payload, + }); + } + cache.db.transaction(() => { + for (const item of preparedWrites) { + cache.write.run(item.file, item.sig, PARSED_CACHE_VERSION, item.payload, now); + } + })(); } catch (error) { if (isNodeSqliteUnavailableError(error)) { reportMissingNodeSqlite(opts?.logLevel, error); @@ -524,3 +545,13 @@ export function writeToCache( } } } + +export function writeToCache( + projectRoot: string, + file: string, + sig: string, + mod: ModuleIndex, + opts?: BuildOptions, +): void { + writeModulesToCache(projectRoot, [{ file, sig, mod }], opts); +} diff --git a/src/indexer/build-cache/project-snapshot.ts b/src/indexer/build-cache/project-snapshot.ts index 41906a6a..c867d569 100644 --- a/src/indexer/build-cache/project-snapshot.ts +++ b/src/indexer/build-cache/project-snapshot.ts @@ -31,11 +31,22 @@ import { type SymbolVisibility, } from "../../graphs/symbol-graph.js"; import { getImplementationFingerprint, normalizeGraphOptions } from "./options.js"; -import { cacheAbsolutePath, cacheRelativePath, cacheRoot } from "./module-cache.js"; +import { cacheAbsolutePath, cacheRelativePath, cacheRoot, type FileSignature } from "./module-cache.js"; import type { ManifestFileEntry } from "./manifest.js"; const SNAPSHOT_SYMBOL_KINDS = new Set(Object.values(SymbolKind)); -const PROJECT_SNAPSHOT_VERSION = 5; +const PROJECT_SNAPSHOT_VERSION = 6; +export const BLOOM_FILTER_SNAPSHOT_VERSION = 2; +export const BLOOM_FILTER_SNAPSHOT_FILENAME = "bloom-filters.json"; + +export type BloomFilterSnapshotPayload = { + version: number; + projectRoot: string; + implementationFingerprint: string; + projectSnapshotIdentity: string; + fileSignatures: Record; + bloomFilters: Record; +}; const BLOOM_FILTER_MIN_SIZE = 1_000; const BLOOM_FILTER_MAX_SIZE = 1_000_000; const BLOOM_FILTER_MIN_HASH_COUNT = 1; @@ -84,6 +95,15 @@ type SerializedBloomFilter = { bitsBase64: string; }; +type SnapshotFileSignature = { + sig: string; + gitSig?: string; +}; + +export type PersistedBloomFilters = { + get: (file: string, signature: Pick) => BloomFilter | undefined; +}; + type SnapshotAnalysisReport = { backend?: BackendReport; graph?: GraphReport; @@ -109,6 +129,7 @@ type ProjectIndexSnapshotPayload = { implementationFingerprint: string; projectFiles?: ProjectFileInfo[]; bloomFilters?: Record; + fileSignatures: Record; analysis?: AnalysisSummary; analysisReport?: SnapshotAnalysisReport; }; @@ -203,6 +224,11 @@ function transformSnapshotPaths( } copy.bloomFilters = bloomFilters; } + const fileSignatures: Record = {}; + for (const [file, signature] of Object.entries(copy.fileSignatures)) { + fileSignatures[transformPath(root, file, toRelative)] = signature; + } + copy.fileSignatures = fileSignatures; return copy; } function migrateProjectSnapshotPayload(value: unknown, currentRoot: string): unknown { @@ -449,46 +475,151 @@ export async function tryLoadProjectIndexSnapshot( } /** - * Load only the bloom-filter section of the last-written project snapshot, without requiring - * the whole snapshot to match this build's files signature or native runtime fingerprint. - * Bloom filters are a pure function of a file's text, so a filter persisted for a given file - * is safe to reuse after its snapshot's root and implementation fingerprints are validated. + * Load only the bloom-filter section of the last-written project snapshot. Hydration checks + * each requested file against its persisted signature before returning a filter, so a stale + * snapshot can still accelerate unchanged files without suppressing newly added references. * Only the bloom section is then read, so a corrupt payload is rejected without walking * `graph.edges` / `modules`. Returns `null` when disk caching is off, `useBloomFilters` is * disabled, or no valid snapshot with bloom data exists. */ +export async function tryLoadProjectSnapshotModules( + projectRoot: string, + opts: BuildOptions | undefined, + fileSignatures: ReadonlyMap>, +): Promise | null> { + if ((opts?.cache ?? "off") !== "disk") return null; + try { + const rawPayload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; + const migratedPayload = migrateProjectSnapshotPayload(rawPayload, projectRoot); + const payload = + migratedPayload && typeof migratedPayload === "object" && !Array.isArray(migratedPayload) + ? transformSnapshotPaths(migratedPayload as ProjectIndexSnapshotPayload, projectRoot, false) + : migratedPayload; + const nativeRuntimeFingerprint = getNativeRuntimeFingerprint(opts?.native); + const implementationFingerprint = getImplementationFingerprint(); + if ( + !isProjectIndexSnapshotPayload(payload) || + !projectRootMatches(projectRoot, payload.projectRoot) || + payload.nativeMode !== normalizedSnapshotNativeMode(opts?.native) || + payload.nativeRuntimeFingerprint !== nativeRuntimeFingerprint || + payload.implementationFingerprint !== implementationFingerprint + ) { + return null; + } + const modules = new Map(); + for (const mod of payload.modules) { + const signature = fileSignatures.get(fileIdentityKey(mod.file)); + const snapshotSignature = payload.fileSignatures[fileIdentityKey(mod.file)]; + if (!signature || !snapshotSignature || !snapshotSignatureMatches(snapshotSignature, signature)) continue; + modules.set(fileIdentityKey(mod.file), mod); + } + return modules; + } catch { + return null; + } +} + export async function tryLoadPersistedBloomFilters( projectRoot: string, opts: BuildOptions | undefined, -): Promise { +): Promise { if ((opts?.cache ?? "off") !== "disk" || (opts?.useBloomFilters ?? true) === false) return null; + try { + const sidecarPath = bloomFilterSnapshotPath(projectRoot, opts); + const sidecarParsed = (await readParsedSnapshot(sidecarPath)).payload; + const sidecarBloom = persistedBloomFiltersFromSidecar(sidecarParsed, projectRoot); + if (sidecarBloom) { + return createPersistedBloomFilters(sidecarBloom.bloomFilters, sidecarBloom.fileSignatures, projectRoot); + } + } catch { + // Fall back to legacy project snapshot payload if sidecar is unavailable or corrupt + } try { const payload = (await readParsedSnapshot(projectSnapshotPath(projectRoot, opts))).payload; const bloomFilters = persistedBloomFiltersFromSnapshot(payload, projectRoot); if (!bloomFilters) return null; - return deserializeBloomFilterCache(bloomFilters, projectRoot); + return createPersistedBloomFilters(bloomFilters.bloomFilters, bloomFilters.fileSignatures, projectRoot); } catch { return null; } } +function persistedBloomFiltersFromSidecar( + value: unknown, + projectRoot: string, +): Pick | null { + if (!value || typeof value !== "object") return null; + const payload = value as Partial; + if ( + payload.version !== BLOOM_FILTER_SNAPSHOT_VERSION || + typeof payload.projectRoot !== "string" || + !projectRootMatches(projectRoot, payload.projectRoot) || + payload.implementationFingerprint !== getImplementationFingerprint() || + typeof payload.projectSnapshotIdentity !== "string" || + !/^[a-f0-9]{64}$/.test(payload.projectSnapshotIdentity) + ) { + return null; + } + const bloomFilters = payload.bloomFilters; + const fileSignatures = payload.fileSignatures; + if (!isSerializedBloomFilterRecord(bloomFilters) || !isSnapshotFileSignatureRecord(fileSignatures)) { + return null; + } + return { bloomFilters, fileSignatures }; +} + /** Light validation for bloom hydration: snapshot version, root identity, and bloom section only. */ function persistedBloomFiltersFromSnapshot( value: unknown, projectRoot: string, -): Record | null { +): Pick | null { const migrated = migrateProjectSnapshotPayload(value, projectRoot); if (!migrated || typeof migrated !== "object") return null; const payload = migrated as Partial; if ( payload.version !== PROJECT_SNAPSHOT_VERSION || typeof payload.projectRoot !== "string" || + !projectRootMatches(projectRoot, payload.projectRoot) || payload.implementationFingerprint !== getImplementationFingerprint() ) { return null; } - if (!isSerializedBloomFilterRecord(payload.bloomFilters)) return null; - return payload.bloomFilters; + const bloomFilters = payload.bloomFilters; + const fileSignatures = payload.fileSignatures; + if (!isSerializedBloomFilterRecord(bloomFilters) || !isSnapshotFileSignatureRecord(fileSignatures)) { + return null; + } + return { bloomFilters, fileSignatures }; +} + +function createPersistedBloomFilters( + bloomFilters: Record, + fileSignatures: Record, + projectRoot: string, +): PersistedBloomFilters { + const filters = deserializeBloomFilterCache(bloomFilters, projectRoot); + const signatures = new Map( + Object.entries(fileSignatures).map(([file, signature]) => [ + fileIdentityKey(cacheAbsolutePath(projectRoot, file)), + signature, + ]), + ); + return { + get: (file, signature) => { + const persistedSignature = signatures.get(fileIdentityKey(file)); + if (!persistedSignature || !snapshotSignatureMatches(persistedSignature, signature)) return undefined; + return filters.get(file); + }, + }; +} + +function snapshotSignatureMatches( + snapshotSignature: SnapshotFileSignature, + currentSignature: Pick, +): boolean { + const matchingGitSignature = + !!snapshotSignature.gitSig && !!currentSignature.gitSig && snapshotSignature.gitSig === currentSignature.gitSig; + return matchingGitSignature || snapshotSignature.sig === currentSignature.sig; } export async function writeProjectIndexSnapshot( @@ -498,6 +629,7 @@ export async function writeProjectIndexSnapshot( filesSignature: string, ): Promise { const projectSnapshotIdentity = createProjectSnapshotIdentity(filesSignature, opts); + const fileSignatures = serializeSnapshotFileSignatures(index.manifestEntries, projectRoot); const serializedBloomFilters = index.bloomFilters ? serializeBloomFilterCache( index.bloomFilters, @@ -519,6 +651,7 @@ export async function writeProjectIndexSnapshot( edges: index.graph.edges, }, modules: [...index.byFile.values()], + fileSignatures, ...(index.languageExtensions ? { languageExtensions: index.languageExtensions } : {}), ...(normalizedSnapshotNativeMode(index.nativeMode) ? { nativeMode: normalizedSnapshotNativeMode(index.nativeMode) } @@ -540,6 +673,30 @@ export async function writeProjectIndexSnapshot( const identity = await snapshotFileIdentity(snapshotPath); setBoundedSnapshotCache(parsedSnapshotCache, snapshotPath, { identity, compressed }); index.projectSnapshotIdentity = projectSnapshotIdentity; + if (serializedBloomFilters) { + try { + const bloomPayload: BloomFilterSnapshotPayload = { + version: BLOOM_FILTER_SNAPSHOT_VERSION, + projectRoot: serializedProjectRoot(projectRoot), + implementationFingerprint: getImplementationFingerprint(), + projectSnapshotIdentity, + fileSignatures, + bloomFilters: serializedBloomFilters, + }; + const bloomPath = bloomFilterSnapshotPath(projectRoot, opts); + const bloomCompressed = brotliCompressSync(JSON.stringify(bloomPayload), { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }); + await writeSnapshotAtomically(bloomPath, bloomCompressed); + const bloomIdentity = await snapshotFileIdentity(bloomPath); + setBoundedSnapshotCache(parsedSnapshotCache, bloomPath, { + identity: bloomIdentity, + compressed: bloomCompressed, + }); + } catch { + // Bloom sidecar write failure must not fail indexing + } + } } catch { delete index.projectSnapshotIdentity; // Snapshot writes are an optimization; cache write failures must not fail indexing. @@ -550,6 +707,15 @@ function projectSnapshotPath(projectRoot: string, opts: BuildOptions | undefined return path.join(cacheRoot(projectRoot, opts), "project-index-snapshot.json"); } +function bloomFilterSnapshotPath(projectRoot: string, opts: BuildOptions | undefined): string { + const root = path.resolve(cacheRoot(projectRoot, opts)); + const snapshotPath = path.resolve(root, BLOOM_FILTER_SNAPSHOT_FILENAME); + if (!isFilePathWithinRoot(root, snapshotPath)) { + throw new Error(`Bloom filter snapshot escaped cache root: ${snapshotPath}`); + } + return snapshotPath; +} + export async function tryLoadDetailedSymbolGraphSnapshot( projectRoot: string, opts: BuildOptions | undefined, @@ -950,6 +1116,7 @@ function isProjectIndexSnapshotPayload(value: unknown): value is ProjectIndexSna payload.graph.edges.every(isGraphEdge) && Array.isArray(payload.modules) && payload.modules.every(isModuleIndex) && + isSnapshotFileSignatureRecord(payload.fileSignatures) && (payload.nativeMode === undefined || isSnapshotNativeMode(payload.nativeMode)) && (payload.bloomFilters === undefined || isSerializedBloomFilterRecord(payload.bloomFilters)) && (payload.analysis === undefined || isAnalysisSummary(payload.analysis)) && @@ -1070,6 +1237,32 @@ function serializeBloomFilterCache( } return Object.keys(serialized).length ? serialized : undefined; } + +function serializeSnapshotFileSignatures( + entries: ProjectIndex["manifestEntries"], + projectRoot: string, +): Record { + const serialized: Record = {}; + for (const [file, entry] of entries ?? []) { + serialized[cacheRelativePath(projectRoot, file)] = { + sig: entry.sig, + ...(entry.gitSig ? { gitSig: entry.gitSig } : {}), + }; + } + return serialized; +} + +function isSnapshotFileSignatureRecord(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + return Object.values(value).every(isSnapshotFileSignature); +} + +function isSnapshotFileSignature(value: unknown): value is SnapshotFileSignature { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const signature = value as Partial; + return typeof signature.sig === "string" && (signature.gitSig === undefined || typeof signature.gitSig === "string"); +} + function deserializeBloomFilterCache( serialized: Record, projectRoot: string, @@ -1112,10 +1305,7 @@ function isSerializedBloomFilter(value: unknown): value is SerializedBloomFilter } catch { return false; } - return ( - decoded.length === expectedBytes && - decoded.toString("base64") === filter.bitsBase64 - ); + return decoded.length === expectedBytes && decoded.toString("base64") === filter.bitsBase64; } function isModuleIndex(value: unknown): value is ModuleIndex { diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 7166253f..e3942814 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -64,11 +64,14 @@ import { tryLoadFromCache, tryLoadPersistedBloomFilters, tryLoadProjectIndexSnapshot, + tryLoadProjectSnapshotModules, verifyManifestEntries, + writeModulesToCache, writeProjectIndexSnapshot, writeToCache, type FileSignature, type ManifestFileEntry, + type PendingModuleCacheWrite, } from "./build-cache.js"; import { cacheRoot } from "./build-cache/module-cache.js"; import { @@ -119,6 +122,7 @@ type IndexedFileGraphContext = { type IndexedFileModuleResult = { module: ModuleIndex; + cacheWrite?: PendingModuleCacheWrite | undefined; graphContext: IndexedFileGraphContext; }; @@ -316,15 +320,17 @@ async function buildIndexedModuleForFile(args: { const sigInfo = args.fileSignatures.get(args.file); const cacheable = !prepared.nativeFallbackReason && !lacksParserContext; + let cacheWrite: PendingModuleCacheWrite | undefined; if (sigInfo && cacheable) { const cacheSig = args.cacheEnabled ? await moduleCacheSignatureForFile(args.file, sigInfo, args.opts) : sigInfo.cacheSig; - writeToCache(args.projectRoot, args.file, cacheSig, mod, args.opts); + cacheWrite = { file: args.file, sig: cacheSig, mod }; } return { module: mod, + ...(cacheWrite !== undefined ? { cacheWrite } : {}), graphContext: { source, sup, @@ -756,7 +762,7 @@ async function buildIndexFromFileListShared( ...(sqlFactCache ? { sqlFactCache } : {}), }); if (bloomFilterCache) { - const persistedFilter = persistedBloomFilters?.get(file); + const persistedFilter = persistedBloomFilters?.get(file, sigInfo); if (persistedFilter) { bloomFilterCache.set(file, persistedFilter); } else { @@ -764,13 +770,14 @@ async function buildIndexFromFileListShared( if (filter) bloomFilterCache.set(file, filter); } } - return [file, mod, edges] as const; + return [file, mod, edges, undefined] as const; } if (fileReport) fileReport.parsed = (fileReport.parsed ?? 0) + 1; const support = supportForFile(file, opts?.languageExtensions); - if (!support) return [file, createEmptyModuleIndex(file), []] as const; + if (!support) return [file, createEmptyModuleIndex(file), [], undefined] as const; ensureBuildProgressStarted(); let graphContext: IndexedFileGraphContext | undefined; + let cacheWrite: PendingModuleCacheWrite | undefined; if (!mod) { const built = await buildIndexedModuleForFile({ file, @@ -791,6 +798,7 @@ async function buildIndexFromFileListShared( }); mod = built.module; graphContext = built.graphContext; + cacheWrite = built.cacheWrite; } else { collectJsonDependencies(mod.imports, jsonDependencies); } @@ -815,11 +823,11 @@ async function buildIndexFromFileListShared( allFiles: normalizedFiles, ...(sqlFactCache ? { sqlFactCache } : {}), }); - return [file, mod ?? createEmptyModuleIndex(file), edges] as const; + return [file, mod ?? createEmptyModuleIndex(file), edges, cacheWrite] as const; } catch (error) { if (isNativeRequiredUnavailableError(error) || isNodeSqliteUnavailableError(error)) throw error; if (isUnsupportedParserInputError(error) || isNonNativeParserUnavailableError(error)) { - return [file, createEmptyModuleIndex(file), []] as const; + return [file, createEmptyModuleIndex(file), [], undefined] as const; } recordFileFailure(report, file, error); logWithLevel(opts?.logLevel, "warn", `Warning: Failed to process file ${file}:`, error); @@ -851,9 +859,14 @@ async function buildIndexFromFileListShared( if (edge.to.type === "file") graph.nodes.add(edge.to.path); } }; - for (const [file, mod, edges] of fileResults) { + const pendingCacheWrites: PendingModuleCacheWrite[] = []; + for (const [file, mod, edges, cacheWrite] of fileResults) { modules.set(fileIdentityKey(file), mod); appendUniqueGraphEdges(edges); + if (cacheWrite) pendingCacheWrites.push(cacheWrite); + } + if (pendingCacheWrites.length) { + writeModulesToCache(projectRoot, pendingCacheWrites, opts); } const workspaceManifestEdges = await collectWorkspaceManifestDependencyEdges( projectRoot, @@ -1434,7 +1447,7 @@ export async function buildProjectIndexIncremental( opts, gitSigMap, cacheEnabled, - needsContentHash: cacheEnabled || useManifest || opts?.cacheStrict === true, + needsContentHash: cacheEnabled || manifestUsed || opts?.cacheStrict === true, concurrency: conc, }); const modules = new Map(); @@ -1479,18 +1492,27 @@ export async function buildProjectIndexIncremental( completeCheckProgress(allFiles.size); return unchangedSnapshot; } + const snapshotModules = cacheEnabled + ? await tryLoadProjectSnapshotModules(projectRoot, opts, fileSignatures) + : null; const persistedBloomFilters = bloomFilterCache ? await tryLoadPersistedBloomFilters(projectRoot, opts) : null; for (const file of allFiles) { if (changedFiles.has(file)) continue; const sigInfo = fileSignatures.get(file)!; - const cacheSig = cacheEnabled ? await moduleCacheSignatureForFile(file, sigInfo, opts) : sigInfo.cacheSig; - const cached = cacheEnabled ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) : null; + let cached: ModuleIndex | null = null; + const snapshotMod = snapshotModules?.get(fileIdentityKey(file)); + if (snapshotMod) { + cached = snapshotMod; + } else if (cacheEnabled) { + const cacheSig = await moduleCacheSignatureForFile(file, sigInfo, opts); + cached = tryLoadFromCache(projectRoot, file, cacheSig, opts, report); + } if (cached) { if (fileReport) fileReport.cached = (fileReport.cached ?? 0) + 1; modules.set(fileIdentityKey(file), cached); collectJsonDependencies(cached.imports, jsonDependencies); if (bloomFilterCache) { - const persistedFilter = persistedBloomFilters?.get(file); + const persistedFilter = persistedBloomFilters?.get(file, sigInfo); if (persistedFilter) { bloomFilterCache.set(file, persistedFilter); } else { @@ -1536,7 +1558,7 @@ export async function buildProjectIndexIncremental( fileSignatures, cacheEnabled, }); - return [file, built.module] as const; + return [file, built.module, built.cacheWrite] as const; } catch (error) { if (isNativeRequiredUnavailableError(error) || isNodeSqliteUnavailableError(error)) throw error; if (isUnsupportedParserInputError(error) || isNonNativeParserUnavailableError(error)) { @@ -1544,7 +1566,7 @@ export async function buildProjectIndexIncremental( } recordFileFailure(report, file, error); logWithLevel(opts?.logLevel, "warn", `Warning: Failed to process file ${file}:`, error); - return [file, createEmptyModuleIndex(file)] as const; + return [file, createEmptyModuleIndex(file), undefined] as const; } finally { if (opts?.onProgress) { opts.onProgress({ @@ -1558,8 +1580,13 @@ export async function buildProjectIndexIncremental( } } }); - for (const [file, mod] of fileResults) { + const pendingIncrementalCacheWrites: PendingModuleCacheWrite[] = []; + for (const [file, mod, cacheWrite] of fileResults) { modules.set(fileIdentityKey(file), mod); + if (cacheWrite) pendingIncrementalCacheWrites.push(cacheWrite); + } + if (pendingIncrementalCacheWrites.length) { + writeModulesToCache(projectRoot, pendingIncrementalCacheWrites, opts); } if (timings) timings.parseMs = Math.round(performance.now() - parseStart); } diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 1c78b03c..6590b33a 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -5,7 +5,13 @@ import fsp from "node:fs/promises"; import fs from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from "node:zlib"; -import { buildProjectIndex, buildProjectIndexIncremental, resolveExport, type BuildReport } from "../src/index.js"; +import { + buildProjectIndex, + buildProjectIndexIncremental, + findReferences, + resolveExport, + type BuildReport, +} from "../src/index.js"; import type { ModuleIndex, ProjectIndex } from "../src/indexer/types.js"; import * as indexer from "../src/indexer.js"; import * as buildCache from "../src/indexer/build-cache.js"; @@ -1739,6 +1745,9 @@ describe("Cache invalidation and strict hashing", () => { await fsp.writeFile(gammaPath, "export const gammaValue = 3;\n", "utf8"); await buildProjectIndex(root, { threads: 2, cache: "disk", useBloomFilters: true }); + const bloomSidecarPath = path.join(root, ".codegraph-cache", "index-v1", "bloom-filters.json"); + expect(await fsp.stat(bloomSidecarPath)).toBeTruthy(); + await fsp.rm(bloomSidecarPath); // Modify only gamma.ts: alpha.ts and beta.ts stay genuine, provable cache hits, but the // snapshot as a whole can no longer be reused wholesale (`changedFiles.size` is nonzero), @@ -1768,6 +1777,79 @@ describe("Cache invalidation and strict hashing", () => { bloomSpy.mockRestore(); }); + it("rejects stale snapshot modules after the manifest has advanced", async () => { + const root = await mkTmpDir("dg-stale-snapshot-modules-"); + const entryPath = path.join(root, "entry.ts"); + await fsp.writeFile(entryPath, "export const staleSnapshotValue = 1;\n", "utf8"); + await buildProjectIndex(root, { threads: 1, cache: "disk", useBloomFilters: true }); + + const snapshotPath = projectSnapshotPathFor(root); + const staleSnapshot = await fsp.readFile(snapshotPath); + await fsp.writeFile(entryPath, "export const currentSnapshotValue = 2;\n", "utf8"); + await buildProjectIndexIncremental(root, { threads: 1, cache: "disk", useBloomFilters: true }); + await fsp.writeFile(snapshotPath, staleSnapshot); + + const recovered = await buildProjectIndexIncremental(root, { + threads: 1, + cache: "disk", + useBloomFilters: true, + }); + const entry = recovered.byFile.get(fileIdentityKey(normalize(entryPath))); + + expect(entry?.locals.some((local) => local.localName === "currentSnapshotValue")).toBe(true); + expect(entry?.locals.some((local) => local.localName === "staleSnapshotValue")).toBe(false); + }); + + it("rejects stale bloom sidecars and recovers semantic references", async () => { + const root = await mkTmpDir("dg-stale-bloom-sidecar-"); + const definitionPath = path.join(root, "definition.ts"); + const consumerPath = path.join(root, "consumer.ts"); + const triggerPath = path.join(root, "trigger.ts"); + await fsp.writeFile(definitionPath, "export class Worker { staleBloomMethod() {} }\n", "utf8"); + await fsp.writeFile( + consumerPath, + 'import { Worker } from "./definition";\nexport const consumer = new Worker().staleBloomMethod();\n', + "utf8", + ); + await fsp.writeFile(triggerPath, "export const trigger = 1;\n", "utf8"); + await buildProjectIndex(root, { threads: 1, cache: "disk", useBloomFilters: true }); + + const snapshotPath = projectSnapshotPathFor(root); + const sidecarPath = path.join(root, ".codegraph-cache", "index-v1", "bloom-filters.json"); + const staleSnapshot = await fsp.readFile(snapshotPath); + const staleSidecar = await fsp.readFile(sidecarPath); + await fsp.writeFile(definitionPath, "export class Worker { currentBloomMethod() {} }\n", "utf8"); + await fsp.writeFile( + consumerPath, + 'import { Worker } from "./definition";\nexport const consumer = new Worker().currentBloomMethod();\n', + "utf8", + ); + await buildProjectIndexIncremental(root, { threads: 1, cache: "disk", useBloomFilters: true }); + await fsp.writeFile(snapshotPath, staleSnapshot); + await fsp.writeFile(sidecarPath, staleSidecar); + await fsp.writeFile(triggerPath, "export const trigger = 2;\n", "utf8"); + + const recovered = await buildProjectIndexIncremental(root, { + threads: 1, + cache: "disk", + useBloomFilters: true, + }); + const definition = recovered.byFile + .get(fileIdentityKey(normalize(definitionPath))) + ?.locals.find((local) => local.localName === "currentBloomMethod"); + if (!definition) throw new Error("Expected current bloom definition"); + + const references = await findReferences(recovered, { def: definition }); + + expect(recovered.bloomFilters?.get(normalize(consumerPath))?.mightContain("currentBloomMethod")).toBe(true); + expect(references.status).toBe("ok"); + if (references.status === "ok") { + expect(references.references.some((reference) => normalize(reference.file) === normalize(consumerPath))).toBe( + true, + ); + } + }); + it("does not hydrate persisted bloom filters when bloom filters are disabled", async () => { const root = await mkTmpDir("dg-snapshot-bloom-disabled-"); const entryPath = path.join(root, "entry.ts"); @@ -1856,7 +1938,7 @@ describe("Cache invalidation and strict hashing", () => { nativeRuntimeFingerprint?: string; implementationFingerprint?: string; }; - expect(rewrittenSnapshot.version).toBe(5); + expect(rewrittenSnapshot.version).toBe(6); expect(initial.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.byFile.has(fileIdentityKey(normalize(entryPath)))).toBe(true); expect(rebuilt.bloomFilters?.get(normalize(entryPath))?.mightContain("versioned")).toBe(true); diff --git a/tests/disk-cache-sqlite.test.ts b/tests/disk-cache-sqlite.test.ts index 5c3cbdec..00a5586c 100644 --- a/tests/disk-cache-sqlite.test.ts +++ b/tests/disk-cache-sqlite.test.ts @@ -6,7 +6,8 @@ import { brotliDecompressSync } from "node:zlib"; import { buildProjectIndex, findDuplicates, type BuildReport } from "../src/index.js"; import { closeDuplicateUnitCacheDatabase } from "../src/duplicates.js"; -import { SqliteDatabase } from "../src/sqlite-driver.js"; +import * as buildCache from "../src/indexer/build-cache.js"; +import { SqliteDatabase, SqliteStatement } from "../src/sqlite-driver.js"; import { mkTmpDir } from "./helpers/filesystem.js"; function cacheDir(root: string): string { @@ -126,6 +127,49 @@ describe("disk cache uses sqlite backend", () => { ).toHaveLength(1); expect(sql.filter((statement) => statement.startsWith("INSERT INTO module_cache"))).toHaveLength(1); }); + it("rolls back a failed module cache batch without a partial row", async () => { + const root = await mkTmpDir("dg-disk-cache-batch-rollback-"); + const firstPath = path.join(root, "first.ts"); + const secondPath = path.join(root, "second.ts"); + await fsp.writeFile(firstPath, "export const first = 1;\n", "utf8"); + await fsp.writeFile(secondPath, "export const second = 2;\n", "utf8"); + const index = await buildProjectIndex(root, { cache: "disk", threads: 1 }); + const first = Array.from(index.byFile.values()).find((mod) => mod.file.endsWith("/first.ts")); + const second = Array.from(index.byFile.values()).find((mod) => mod.file.endsWith("/second.ts")); + if (!first || !second) throw new Error("missing seeded modules"); + + const db = new DatabaseSync(moduleCacheDbPath(root)); + db.exec("DELETE FROM module_cache;"); + db.close(); + + const originalRun = SqliteStatement.prototype.run; + let cacheWrites = 0; + const runSpy = vi.spyOn(SqliteStatement.prototype, "run").mockImplementation(function ( + this: SqliteStatement, + ...params + ) { + if (params.length === 5 && params[2] === 4) { + cacheWrites++; + if (cacheWrites === 2) throw new Error("simulated aborted cache batch"); + } + return originalRun.call(this, ...params); + }); + try { + buildCache.writeModulesToCache( + root, + [ + { file: firstPath, sig: "first-signature", mod: first }, + { file: secondPath, sig: "second-signature", mod: second }, + ], + { cache: "disk" }, + ); + } finally { + runSpy.mockRestore(); + } + + expect(cacheWrites).toBe(2); + expect(readRowCount(moduleCacheDbPath(root), "SELECT COUNT(*) AS count FROM module_cache")).toBe(0); + }); it("prunes module cache rows for files outside the successful manifest", async () => { const root = await mkTmpDir("dg-disk-cache-prune-");