Fix/buck2 isolation dir name (#1913)

* fix(orion): pass buck2 isolation dir as name instead of path

Signed-off-by: jerry <1772030600@qq.com>

* perf(orion,scorpio): reduce cold FUSE metadata latency for buck2

Signed-off-by: jerry <1772030600@qq.com>

* chore: add in-code TODOs for remaining FUSE metadata work

Signed-off-by: jerry <1772030600@qq.com>

---------

Signed-off-by: jerry <1772030600@qq.com>

Author: jerry609

common/src/config/mod.rs

@@ -647,8 +647,6 @@ pub struct BuildConfig {
     pub orion_server: String,
     #[serde(default)]
     pub orion_preheat_shallow_depth: usize,
-    #[serde(default)]
-    pub orion_buck2_isolation_dir_base: Option<String>,
 }
 
 /// Orion Server configuration (flat structure)

config/config-test.toml

@@ -87,11 +87,7 @@ orion_server = "https://orion.gitmega.com"
 
 # Orion worker: shallow preheat depth before buck2 cells/audit.
 # 0 means disabled.
-orion_preheat_shallow_depth = 0
-
-# Orion worker: base directory for buck2 --isolation-dir.
-# Empty means use the system temp directory.
-orion_buck2_isolation_dir_base = ""
+orion_preheat_shallow_depth = 3
 
 
 [pack]

config/config-workflow.toml

@@ -83,11 +83,7 @@ orion_server = "https://orion.gitmega.com"
 
 # Orion worker: shallow preheat depth before buck2 cells/audit.
 # 0 means disabled.
-orion_preheat_shallow_depth = 0
-
-# Orion worker: base directory for buck2 --isolation-dir.
-# Empty means use the system temp directory.
-orion_buck2_isolation_dir_base = ""
+orion_preheat_shallow_depth = 3
 
 
 [pack]

config/config.toml

@@ -88,11 +88,7 @@ orion_server = "https://orion.gitmega.com"
 
 # Orion worker: shallow preheat depth before buck2 cells/audit.
 # 0 means disabled.
-orion_preheat_shallow_depth = 0
-
-# Orion worker: base directory for buck2 --isolation-dir.
-# Empty means use the system temp directory.
-orion_buck2_isolation_dir_base = ""
+orion_preheat_shallow_depth = 3
 
 
 [pack]

orion/.env.example

@@ -68,5 +68,4 @@ ORION_WORKER_START_SCORPIO=true
 # ---------------------------------------------------------------------------
 # Buck2 Configuration
 # ---------------------------------------------------------------------------
-BUCK2_ISOLATION_DIR=/tmp/buck2-isolation
 

orion/Dockerfile

@@ -171,9 +171,7 @@ ENV PATH="/app/bin:/usr/local/bin:${PATH}" \
     RUST_LOG=info \
     # Scorpio defaults
     SCORPIO_STORE_PATH=/data/scorpio/store \
-    SCORPIO_WORKSPACE=/workspace/mount \
-    # Buck2 defaults
-    BUCK2_ISOLATION_DIR=/tmp/buck2-isolation
+    SCORPIO_WORKSPACE=/workspace/mount
 
 WORKDIR /workspace
 

orion/src/buck_controller.rs

@@ -35,6 +35,7 @@ static PROJECT_ROOT: Lazy<String> =
     Lazy::new(|| std::env::var("BUCK_PROJECT_ROOT").expect("BUCK_PROJECT_ROOT must be set"));
 
 const MOUNT_TIMEOUT_SECS: u64 = 7200;
+const DEFAULT_PREHEAT_SHALLOW_DEPTH: usize = 3;
 static BUILD_CONFIG: Lazy<Option<BuildConfig>> = Lazy::new(load_build_config);
 
 /// Mounts filesystem via remote API for repository access.
@@ -411,6 +412,9 @@ async fn unmount_fs(repo: &str, cl: Option<&str>) -> Result<bool, Box<dyn Error
 
 /// Buck2 targets stats every directory, which is slow on FUSE.
 /// We pre-warm metadata with `ls -lR` to reduce statx latency.
+/// TODO(perf): Replace this full-tree walk with a targeted preheat plan that
+/// only touches changed paths + ancestors and buck-critical files
+/// (PACKAGE/BUCK/.buckconfig) to avoid duplicated tree scans.
 /// TODO: Rewrite the targets logic in the monolith.
 fn preheat(repo_path: &Path) -> anyhow::Result<()> {
     let preheat_status = std::process::Command::new("ls")
@@ -485,7 +489,7 @@ fn preheat_shallow_depth() -> usize {
 
     build_config()
         .map(|config| config.orion_preheat_shallow_depth)
-        .unwrap_or(0)
+        .unwrap_or(DEFAULT_PREHEAT_SHALLOW_DEPTH)
 }
 
 fn parse_env_usize(key: &str) -> Option<usize> {
@@ -501,21 +505,6 @@ fn parse_env_usize(key: &str) -> Option<usize> {
     }
 }
 
-fn parse_env_path(key: &str) -> Option<PathBuf> {
-    match std::env::var(key) {
-        Ok(value) => {
-            let trimmed = value.trim();
-            if trimmed.is_empty() {
-                tracing::warn!("Empty {key} provided, ignoring.");
-                None
-            } else {
-                Some(PathBuf::from(trimmed))
-            }
-        }
-        Err(_) => None,
-    }
-}
-
 fn build_config() -> Option<&'static BuildConfig> {
     BUILD_CONFIG.as_ref()
 }
@@ -558,63 +547,26 @@ fn resolve_config_path() -> Option<PathBuf> {
     None
 }
 
-fn buck2_isolation_dir_base() -> (PathBuf, bool) {
-    if let Some(path) = parse_env_path("ORION_BUCK2_ISOLATION_DIR_BASE") {
-        return (path, true);
-    }
-
-    if let Some(path) = parse_env_path("MEGA_BUILD__ORION_BUCK2_ISOLATION_DIR_BASE") {
-        return (path, true);
-    }
-
-    if let Some(config) = build_config()
-        && let Some(path) = config.orion_buck2_isolation_dir_base.as_ref()
-    {
-        let trimmed = path.trim();
-        if !trimmed.is_empty() {
-            return (PathBuf::from(trimmed), true);
-        }
-    }
-
-    (std::env::temp_dir(), false)
-}
-
-/// Derive a stable per-mount buck2 isolation directory and ensure it exists.
-fn buck2_isolation_dir(repo_path: &Path) -> anyhow::Result<PathBuf> {
+/// Derive a stable per-mount buck2 isolation directory name.
+///
+/// Buck2 `--isolation-dir` expects a plain directory **name** (no path
+/// separators).  Buck2 itself stores daemon state under
+/// `<project_root>/.buck2/<isolation_dir>/`, so we only need to return a
+/// unique name – not a full path.
+fn buck2_isolation_dir(repo_path: &Path) -> anyhow::Result<String> {
     let digest = ring::digest::digest(
         &ring::digest::SHA256,
         repo_path.to_string_lossy().as_bytes(),
     );
     let suffix = &hex::encode(digest.as_ref())[..16];
-    let (base, from_env) = buck2_isolation_dir_base();
-    let mut path = base.join(format!("buck2-isolation-{suffix}"));
-    if let Err(err) = std::fs::create_dir_all(&path) {
-        if from_env {
-            tracing::warn!(
-                "Failed to create buck2 isolation dir {path:?} from ORION_BUCK2_ISOLATION_DIR_BASE: {err}. Falling back to temp dir."
-            );
-            let fallback = std::env::temp_dir();
-            path = fallback.join(format!("buck2-isolation-{suffix}"));
-            std::fs::create_dir_all(&path)
-                .map_err(|err| anyhow!("Failed to create buck2 isolation dir {path:?}: {err}"))?;
-        } else {
-            return Err(anyhow!(
-                "Failed to create buck2 isolation dir {path:?}: {err}"
-            ));
-        }
-    }
-    Ok(path)
+    Ok(format!("buck2-isolation-{suffix}"))
 }
 
 /// Get target of a specific repo under tmp directory.
 fn get_repo_targets(file_name: &str, repo_path: &Path) -> anyhow::Result<Targets> {
     const MAX_ATTEMPTS: usize = 2;
     let jsonl_path = PathBuf::from(repo_path).join(file_name);
     let isolation_dir = buck2_isolation_dir(repo_path)?;
-    let isolation_dir = isolation_dir
-        .to_str()
-        .ok_or_else(|| anyhow!("Invalid isolation dir path: {isolation_dir:?}"))?
-        .to_string();
 
     preheat(repo_path)?;
 
@@ -669,12 +621,7 @@ async fn get_build_targets(
 
     preheat_shallow(&mount_path, preheat_shallow_depth())?;
     let mut buck2 = Buck2::with_root("buck2".to_string(), mount_path.clone());
-    buck2.set_isolation_dir(
-        buck2_isolation_dir(&mount_path)?
-            .to_str()
-            .ok_or_else(|| anyhow!("Invalid isolation dir path: {mount_path:?}"))?
-            .to_string(),
-    );
+    buck2.set_isolation_dir(buck2_isolation_dir(&mount_path)?);
     let mut cells = CellInfo::parse(
         &buck2
             .cells()
@@ -894,10 +841,6 @@ pub async fn build(
         // This ensures buck2 uses the sub-project's .buckconfig and PACKAGE files.
         let project_root = PathBuf::from(&mount_point).join(repo_prefix);
         let isolation_dir = buck2_isolation_dir(&project_root)?;
-        let isolation_dir = isolation_dir
-            .to_str()
-            .ok_or_else(|| anyhow!("Invalid isolation dir path: {isolation_dir:?}"))?
-            .to_string();
         let mut cmd = Command::new("buck2");
         cmd.args(["--isolation-dir", &isolation_dir]);
         let cmd = cmd

scorpio/src/antares/fuse.rs

@@ -6,7 +6,7 @@ use libfuse_fs::{
 };
 use tokio::task::JoinHandle;
 
-use crate::server::mount_filesystem;
+use crate::server::{mount_filesystem, mount_filesystem_with_antares_cache};
 
 /// Antares union-fs wrapper: dicfuse lower + passthrough upper/CL.
 pub struct AntaresFuse {
@@ -94,7 +94,8 @@ impl AntaresFuse {
 
         let overlay = self.build_overlay().await?;
         let logfs = LoggingFileSystem::new(overlay);
-        let handle = mount_filesystem(logfs, self.mountpoint.as_os_str()).await;
+        let handle =
+            mount_filesystem_with_antares_cache(logfs, self.mountpoint.as_os_str(), true).await;
 
         // Spawn background task to run the FUSE session
         let fuse_task = tokio::spawn(async move {

scorpio/src/dicfuse/async_io.rs

@@ -174,6 +174,8 @@ impl Filesystem for Dicfuse {
         let child = match child {
             Some(v) => v,
             None => {
+                // TODO(perf): add short-lived negative lookup cache for ENOENT
+                // to avoid repeated misses for Buck2 probe paths.
                 return Err(std::io::Error::from_raw_os_error(libc::ENOENT).into());
             }
         };

scorpio/src/server/mod.rs

@@ -59,10 +59,36 @@ use rfuse3::{
     MountOptions,
 };
 
+#[cfg(target_os = "linux")]
+// TODO(perf): tune these values with production lookup metrics and make them
+// configurable per mount role (interactive vs build-heavy workers).
+const ANTARES_FUSE_CACHE_MOUNT_OPTIONS: &str =
+    "kernel_cache,auto_cache,entry_timeout=60,attr_timeout=60,negative_timeout=10";
+
+#[cfg(not(target_os = "linux"))]
+const ANTARES_FUSE_CACHE_MOUNT_OPTIONS: &str = "";
+
+fn apply_antares_cache_mount_options(options: &mut MountOptions) {
+    if !ANTARES_FUSE_CACHE_MOUNT_OPTIONS.is_empty() {
+        options.custom_options(ANTARES_FUSE_CACHE_MOUNT_OPTIONS);
+    }
+}
+
 #[allow(unused)]
 pub async fn mount_filesystem<F: Filesystem + std::marker::Sync + Send + 'static>(
     fs: F,
     mountpoint: &OsStr,
+) -> MountHandle {
+    mount_filesystem_with_antares_cache(fs, mountpoint, false).await
+}
+
+#[allow(unused)]
+pub async fn mount_filesystem_with_antares_cache<
+    F: Filesystem + std::marker::Sync + Send + 'static,
+>(
+    fs: F,
+    mountpoint: &OsStr,
+    enable_antares_cache: bool,
 ) -> MountHandle {
     if let Err(e) = env_logger::try_init() {
         if !e.to_string().contains("initialized") {
@@ -96,6 +122,9 @@ pub async fn mount_filesystem<F: Filesystem + std::marker::Sync + Send + 'static
     let mut mount_options = MountOptions::default();
     // .allow_other(true)
     mount_options.force_readdir_plus(true).uid(uid).gid(gid);
+    if enable_antares_cache {
+        apply_antares_cache_mount_options(&mut mount_options);
+    }
 
     let session = Session::<F>::new(mount_options);
     session.mount(fs, mount_path).await.unwrap()