feat(symgraph): improve ingestion pipeline for large repositories.

Cargo.toml

@@ -66,6 +66,7 @@ sha2 = "0.11"
 hex = "0.4"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+indicatif = "0.17"
 
 [features]
 default = ["sqlite"]

src/cli/commands.rs

@@ -5,9 +5,11 @@ use tracing::info;
 
 use crate::context::{format_context_markdown, ContextBuilder, ContextOptions};
 use crate::db::Database;
-use crate::{index_codebase, IndexConfig};
+use crate::IndexConfig;
 
-use super::db_utils::{canonicalize_path, open_project_database, prune_cache, resolve_db};
+use super::db_utils::{
+    canonicalize_path, open_project_database, prune_cache, rebuild_project_database, resolve_db,
+};
 
 /// Index a codebase at the given path
 pub fn index_command(path: &str) -> Result<()> {
@@ -16,11 +18,11 @@ pub fn index_command(path: &str) -> Result<()> {
 
     let config = IndexConfig {
         root: project_root.clone(),
+        show_progress: true,
         ..Default::default()
     };
 
-    let stats = index_codebase(&mut db, &config)?;
-
+    let stats = rebuild_project_database(&mut db, &config)?;
     println!("\nIndexing complete!");
     println!("  Files indexed: {}", stats.files);
     println!("  Symbols found: {}", stats.nodes);

src/cli/db_utils.rs

@@ -13,13 +13,18 @@
 //! the caller via `--in-memory` / `SYMGRAPH_IN_MEMORY`.
 
 use anyhow::{Context, Result};
-use std::path::PathBuf;
-use std::process::Command;
+use std::{
+    path::{Path, PathBuf},
+    process::{self, Command},
+    time::{SystemTime, UNIX_EPOCH},
+};
 
 use crate::db::Database;
+use crate::{build_full_index, IndexConfig, IndexingStats};
 
 const DB_DIR: &str = ".symgraph";
 const DB_FILE: &str = "index.db";
+const SHADOW_DB_PREFIX: &str = "index.shadow";
 
 /// On-disk storage strategy.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -207,6 +212,77 @@ pub fn open_project_database(project_root: &str) -> Result<Database> {
     Database::open(&resolved.path)
 }
 
+/// Open a fresh shadow database in the project index directory.
+pub fn open_shadow_database(project_root: &str) -> Result<Database> {
+    let shadow_path = shadow_database_path(project_root)?;
+    Database::cleanup_on_disk_path(&shadow_path)?;
+    Database::open(&shadow_path)
+}
+
+/// Best-effort cleanup for a shadow database path and its SQLite sidecars.
+pub fn cleanup_shadow_database_path(path: &Path) -> Result<()> {
+    Database::cleanup_on_disk_path(path)
+}
+
+/// Best-effort cleanup for a shadow database handle and its SQLite sidecars.
+pub fn cleanup_shadow_database(db: &Database) -> Result<()> {
+    match db.path() {
+        Some(path) => cleanup_shadow_database_path(path),
+        None => Ok(()),
+    }
+}
+
+/// Flush, close, and atomically swap a prepared shadow database into place.
+pub fn swap_shadow_database(live_db: &mut Database, shadow_db: Database) -> Result<()> {
+    let shadow_path = shadow_db.prepare_for_swap()?;
+    live_db.replace_with_shadow(&shadow_path)
+}
+
+/// Build a full shadow index and atomically swap it into the live database handle.
+pub fn rebuild_project_database(
+    live_db: &mut Database,
+    config: &IndexConfig,
+) -> Result<IndexingStats> {
+    let mut shadow_db = open_shadow_database(&config.root)?;
+    let shadow_path = shadow_db
+        .path()
+        .map(Path::to_path_buf)
+        .context("shadow database is not file-backed")?;
+
+    match build_full_index(&mut shadow_db, config) {
+        Ok(stats) => {
+            if let Err(err) = swap_shadow_database(live_db, shadow_db) {
+                let _ = cleanup_shadow_database_path(&shadow_path);
+                Err(err)
+            } else {
+                Ok(stats)
+            }
+        }
+        Err(err) => {
+            let _ = cleanup_shadow_database_path(&shadow_path);
+            Err(err)
+        }
+    }
+}
+
+fn shadow_database_path(project_root: &str) -> Result<PathBuf> {
+    // Co-locate the shadow with the resolved live DB so the atomic rename in
+    // `replace_with_shadow` stays within one directory (and filesystem). The
+    // live location follows storage resolution (git-dir / cache / .symgraph),
+    // so we can't assume `.symgraph/` here.
+    let live = resolve_db(project_root)?.path;
+    let dir = live
+        .parent()
+        .context("resolved database path has no parent directory")?;
+    std::fs::create_dir_all(dir)
+        .with_context(|| format!("creating index directory {}", dir.display()))?;
+    let nonce = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .context("System clock is before Unix epoch")?
+        .as_nanos();
+    Ok(dir.join(format!("{SHADOW_DB_PREFIX}.{}.{}.db", process::id(), nonce)))
+}
+
 /// Canonicalize and validate a path
 pub fn canonicalize_path(path: &str) -> Result<String> {
     let canonical = std::path::Path::new(path)