From db8eac6b62ce0abef68fa82553762e57bb133c99 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sun, 12 Apr 2026 18:45:42 -0600 Subject: [PATCH 1/2] perf(incremental): batch WAL checkpoints and fix native CFG bulk insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace per-feature wal_checkpoint(TRUNCATE) calls (~68ms each × 3-4 features) with a single upfront FULL checkpoint before analyses and one TRUNCATE after all analyses complete. suspendJsDb/resumeJsDb callbacks become no-ops, eliminating ~200-270ms of WAL checkpoint overhead on incremental rebuilds with the native engine. Also fix bulkInsertCfg silently failing due to napi-rs null→u32 conversion error: pass undefined (maps to Option::None) instead of null for optional CfgBlockRow fields (startLine, endLine, label). --- src/domain/graph/builder/pipeline.ts | 58 +++++++++++++--------------- src/features/cfg.ts | 6 +-- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index ea67bf4a..e3dd50ff 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -491,20 +491,13 @@ async function runPostNativeAnalysis( } } - // Wire up WAL checkpoint callbacks for the analysis engine + // Flush JS WAL pages once so Rust can see them, then no-op callbacks. + // Previously each feature called wal_checkpoint(TRUNCATE) individually + // (~68ms each × 3-4 features). One PASSIVE checkpoint suffices. if (ctx.nativeDb && ctx.engineOpts) { - ctx.engineOpts.suspendJsDb = () => { - ctx.db.pragma('wal_checkpoint(TRUNCATE)'); - }; - ctx.engineOpts.resumeJsDb = () => { - try { - ctx.nativeDb?.exec('PRAGMA wal_checkpoint(TRUNCATE)'); - } catch (e) { - debug( - `resumeJsDb: WAL checkpoint failed (nativeDb may already be closed): ${toErrorMessage(e)}`, - ); - } - }; + ctx.db.pragma('wal_checkpoint(FULL)'); + ctx.engineOpts.suspendJsDb = () => {}; + ctx.engineOpts.resumeJsDb = () => {}; } try { @@ -532,7 +525,9 @@ async function runPostNativeAnalysis( warn(`Analysis phases failed after native build: ${toErrorMessage(err)}`); } - // Close nativeDb after analyses + // Close nativeDb after analyses — TRUNCATE checkpoint flushes all Rust + // WAL writes so JS and external readers can see them. Runs once after + // all analysis features complete (not per-feature). if (ctx.nativeDb) { try { ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)'); @@ -753,25 +748,20 @@ async function runPipelineStages(ctx: PipelineContext): Promise { await buildEdges(ctx); await buildStructure(ctx); - // Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow) - // which use suspendJsDb/resumeJsDb WAL checkpoint before native writes. + // Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow). // Skip for small incremental builds — same rationale as insertNodes above. + // + // Perf: do ONE upfront PASSIVE checkpoint to flush JS WAL pages so Rust + // can see the latest rows, then make suspendJsDb/resumeJsDb no-ops. + // Previously each feature called wal_checkpoint(TRUNCATE) individually + // (~68ms each × 3-4 features = ~200-270ms overhead on incremental builds). if (ctx.nativeAvailable && !smallIncremental) { reopenNativeDb(ctx, 'analyses'); if (ctx.nativeDb && ctx.engineOpts) { + ctx.db.pragma('wal_checkpoint(FULL)'); ctx.engineOpts.nativeDb = ctx.nativeDb; - ctx.engineOpts.suspendJsDb = () => { - ctx.db.pragma('wal_checkpoint(TRUNCATE)'); - }; - ctx.engineOpts.resumeJsDb = () => { - try { - ctx.nativeDb?.exec('PRAGMA wal_checkpoint(TRUNCATE)'); - } catch (e) { - debug( - `resumeJsDb: WAL checkpoint failed (nativeDb may already be closed): ${toErrorMessage(e)}`, - ); - } - }; + ctx.engineOpts.suspendJsDb = () => {}; + ctx.engineOpts.resumeJsDb = () => {}; } if (!ctx.nativeDb && ctx.engineOpts) { ctx.engineOpts.nativeDb = undefined; @@ -782,11 +772,15 @@ async function runPipelineStages(ctx: PipelineContext): Promise { await runAnalyses(ctx); - // Keep nativeDb open through finalize so persistBuildMetadata, advisory - // checks, and count queries use the native path. closeDbPair inside - // finalize handles both connections. Refresh the JS db so it has a - // valid page cache in case finalize falls back to JS paths (#751). + // Flush Rust WAL writes (AST, complexity, CFG, dataflow) so the JS + // connection and any post-build readers can see them. One TRUNCATE + // here replaces the N per-feature resumeJsDb checkpoints (#checkpoint-opt). if (ctx.nativeDb) { + try { + ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + } catch (e) { + debug(`post-analyses WAL checkpoint failed: ${toErrorMessage(e)}`); + } refreshJsDb(ctx); } diff --git a/src/features/cfg.ts b/src/features/cfg.ts index 65bc745a..7736c741 100644 --- a/src/features/cfg.ts +++ b/src/features/cfg.ts @@ -404,9 +404,9 @@ export async function buildCFGData( blocks: cfg.blocks.map((b) => ({ index: b.index, blockType: b.type, - startLine: b.startLine ?? null, - endLine: b.endLine ?? null, - label: b.label ?? null, + startLine: b.startLine ?? undefined, + endLine: b.endLine ?? undefined, + label: b.label ?? undefined, })), edges: (cfg.edges || []).map((e) => ({ sourceIndex: e.sourceIndex, From c6f28a801612874dcdceb73992d772496612d99a Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sun, 12 Apr 2026 20:11:06 -0600 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20correct=20PASSIVE=E2=86=92FULL=20in?= =?UTF-8?q?=20WAL=20checkpoint=20comments=20(#917)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/domain/graph/builder/pipeline.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index e3dd50ff..0d85751b 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -493,7 +493,7 @@ async function runPostNativeAnalysis( // Flush JS WAL pages once so Rust can see them, then no-op callbacks. // Previously each feature called wal_checkpoint(TRUNCATE) individually - // (~68ms each × 3-4 features). One PASSIVE checkpoint suffices. + // (~68ms each × 3-4 features). One FULL checkpoint suffices. if (ctx.nativeDb && ctx.engineOpts) { ctx.db.pragma('wal_checkpoint(FULL)'); ctx.engineOpts.suspendJsDb = () => {}; @@ -751,7 +751,7 @@ async function runPipelineStages(ctx: PipelineContext): Promise { // Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow). // Skip for small incremental builds — same rationale as insertNodes above. // - // Perf: do ONE upfront PASSIVE checkpoint to flush JS WAL pages so Rust + // Perf: do ONE upfront FULL checkpoint to flush JS WAL pages so Rust // can see the latest rows, then make suspendJsDb/resumeJsDb no-ops. // Previously each feature called wal_checkpoint(TRUNCATE) individually // (~68ms each × 3-4 features = ~200-270ms overhead on incremental builds).