[turbopack] Retain fewer stale cache versions and use a TTL

packages/next/src/lib/turbopack-cache-seed.ts

@@ -76,10 +76,10 @@ function findSeedSource(
   const currentWorktree = path.resolve(worktreeInfo.worktreeRoot)
 
   // We are going to find the best candidate worktree
-  // based on the newest mtime of the CURRENT file in the cache directory.
+  // based on the most recently written cache directory.
   // We only look at our version
 
-  let best: { versionDir: string; mtimeMs: number } | undefined
+  let best: { versionDir: string; commitTimeMs: number } | undefined
   for (const root of [
     worktreeInfo.mainRepoRoot,
     ...listLinkedWorktreeRoots(worktreeInfo.mainRepoRoot),
@@ -93,21 +93,31 @@ function findSeedSource(
       'turbopack',
       version
     )
-    const mtimeMs = currentMtimeMs(versionDir)
-    if (mtimeMs === undefined) continue
-    if (!best || mtimeMs > best.mtimeMs) {
-      best = { versionDir, mtimeMs }
+    const commitTimeMs = currentCommitTimeMs(versionDir)
+    if (commitTimeMs === undefined) continue
+    if (!best || commitTimeMs > best.commitTimeMs) {
+      best = { versionDir, commitTimeMs }
     }
   }
   return best?.versionDir
 }
 
-function currentMtimeMs(versionDir: string): number | undefined {
+// When the cache in `versionDir` was last committed to, in epoch milliseconds, or undefined if
+// there is no usable cache there. Read from the `commit_time` the persistence layer records in
+// CURRENT.
+function currentCommitTimeMs(versionDir: string): number | undefined {
+  let commitTime
   try {
-    return fs.statSync(path.join(versionDir, 'CURRENT')).mtimeMs
+    commitTime = JSON.parse(
+      fs.readFileSync(path.join(versionDir, 'CURRENT'), 'utf8')
+    ).commit_time
   } catch {
+    // missing, unreadable, or not valid JSON - treat it as not a seed candidate
     return undefined
   }
+  if (typeof commitTime !== 'string') return undefined
+  const parsed = Date.parse(commitTime)
+  return Number.isNaN(parsed) ? undefined : parsed
 }
 
 function dirHasEntries(dir: string): boolean {

turbopack/crates/turbo-persistence/Cargo.toml

@@ -20,7 +20,7 @@ crc32fast = { workspace = true }
 dashmap = { workspace = true}
 either = { workspace = true }
 fs-err = { workspace = true }
-jiff = "0.2.10"
+jiff = { version = "0.2.10", features = ["serde"] }
 lzzzz = { workspace = true }
 memmap2 = "0.9.5"
 nohash-hasher = { workspace = true }
@@ -31,6 +31,8 @@ postcard = { workspace = true, features = ["alloc", "use-std"] }
 zerocopy = { version = "0.8", features = ["derive"] }
 quick_cache = { workspace = true }
 rustc-hash = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 smallvec = { workspace = true }
 thread_local = { workspace = true }
 tracing = { workspace = true }

turbopack/crates/turbo-persistence/README.md

@@ -12,7 +12,7 @@ It supports having multiple key families, which are stored in separate files, bu
 
 ## On disk format
 
-There is a single `CURRENT` file which stores the latest committed sequence number.
+There is a single `CURRENT` file, a small JSON object holding the latest committed sequence number (`max_sequence_number`) and when that commit happened (`commit_time`). The commit time is stored in the file rather than taken from its mtime so that it survives the directory being copied or restored. External tools read this file, so its field names are a stable contract.
 
 All other files have a sequence number as file name, e. g. `0000123.sst`. All files are immutable once their sequence number is <= the committed sequence number. But they might be deleted when they are superseded by other committed files.
 

turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs

@@ -24,6 +24,7 @@ use turbo_persistence::{
     BLOCK_HEADER_SIZE, checksum_block,
     meta_file::MetaFile,
     mmap_helper::advise_mmap_for_persistence,
+    read_current_version,
     sst_filter::SstFilter,
     static_sorted_file::{
         BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH,
@@ -211,9 +212,9 @@ fn format_bytes(bytes: u64) -> String {
 /// and apply SstFilter to skip superseded entries.
 fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
     // Read the CURRENT sequence number — only files with seq <= current are valid.
-    let current: u32 = File::open(db_path.join("CURRENT"))?
-        .read_u32::<BE>()
-        .context("Failed to read CURRENT file")?;
+    let current = read_current_version(db_path)?
+        .context("CURRENT file is missing")?
+        .max_sequence_number;
 
     // Read .del files to find sequences that were deleted but not yet cleaned up.
     let mut deleted_seqs: HashSet<u32> = HashSet::new();

turbopack/crates/turbo-persistence/src/db.rs

@@ -2,7 +2,7 @@ use std::{
     borrow::Cow,
     collections::HashSet,
     fmt::Display,
-    io::{BufWriter, Write},
+    io::{BufWriter, ErrorKind, Write},
     mem::take,
     ops::RangeInclusive,
     path::{Path, PathBuf},
@@ -20,6 +20,7 @@ use jiff::Timestamp;
 use memmap2::Mmap;
 use nohash_hasher::BuildNoHashHasher;
 use parking_lot::{Mutex, RwLock};
+use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
 use tracing::span::EnteredSpan;
 
@@ -159,24 +160,61 @@ impl WriteOperationGuard<'_> {
     }
 }
 
+/// The contents of the `CURRENT` file: which sequence number is committed, and when that commit
+/// happened.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CurrentDbVersion {
+    /// The highest sequence number that is part of the committed database.
+    pub max_sequence_number: u32,
+    /// When this database was last committed to.
+    pub commit_time: Timestamp,
+}
+
+/// Reads the `CURRENT` file in the database directory `path`.
+///
+/// Returns `Ok(None)` if the file doesn't exist, which for a writable database means "not
+/// initialized yet".
+pub fn read_current_version(path: &Path) -> Result<Option<CurrentDbVersion>> {
+    let current_path = path.join("CURRENT");
+    let content = match fs::read(&current_path) {
+        Ok(content) => content,
+        Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None),
+        Err(e) => return Err(e).context("Failed to read CURRENT file"),
+    };
+
+    serde_json::from_slice::<CurrentDbVersion>(&content)
+        .with_context(|| {
+            format!(
+                "CURRENT file at {} is corrupt ({} bytes)",
+                current_path.display(),
+                content.len()
+            )
+        })
+        .map(Some)
+}
+
 /// Durably and atomically updates the `CURRENT` file in the database directory `path` to point at
-/// `seq`.
+/// `seq`, stamping it as used now.
 ///
-/// The write is made atomic by writing `seq` to a temporary `CURRENT.next` file, flushing it, and
-/// then `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and
-/// replaces the destination on Windows, so a concurrent or crashing writer can never observe a
-/// torn `CURRENT` (in-place overwrites, by contrast, can leave a partially-written value on a
-/// crash mid-write). After the rename we fsync the directory so the new `CURRENT` → inode mapping
-/// survives a crash.
+/// The write is made atomic by writing to a temporary `CURRENT.next` file, flushing it, and then
+/// `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and replaces the
+/// destination on Windows, so a concurrent or crashing writer can never observe a torn `CURRENT`
+/// (in-place overwrites, by contrast, can leave a partially-written value on a crash mid-write).
+/// After the rename we fsync the directory so the new `CURRENT` → inode mapping survives a crash.
 fn commit_current(path: &Path, seq: u32) -> Result<()> {
+    let version: &CurrentDbVersion = &CurrentDbVersion {
+        max_sequence_number: seq,
+        commit_time: Timestamp::now(),
+    };
+    let mut contents =
+        serde_json::to_string(version).context("Failed to serialize the CURRENT file")?;
+    contents.push('\n');
     let next_path = path.join("CURRENT.next");
     let mut next_file = File::create(&next_path)?;
-    next_file.write_u32::<BE>(seq)?;
+    next_file.write_all(contents.as_bytes())?;
     next_file.sync_data()?;
     drop(next_file);
-
     fs::rename(&next_path, path.join("CURRENT"))?;
-
     // Fsync the directory. This is the single durability barrier for a commit: by the time we get
     // here every file created earlier in the commit (SST/meta/blob and any `.del` file) already
     // exists, so this one fsync flushes *all* of their directory entries together with the CURRENT
@@ -459,7 +497,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
                 Ok(())
             }
             Err(e) => {
-                if !read_only && e.kind() == std::io::ErrorKind::NotFound {
+                if !read_only && e.kind() == ErrorKind::NotFound {
                     self.create_and_init_directory()
                         .context("Creating and initializing persistence directory failed")?;
                     Ok(())
@@ -479,18 +517,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
     /// Loads an existing database directory and performs cleanup if necessary.
     fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> {
         let mut meta_files = Vec::new();
-        let mut current_file = match File::open(self.path.join("CURRENT")) {
-            Ok(file) => file,
-            Err(e) => {
-                if !read_only && e.kind() == std::io::ErrorKind::NotFound {
-                    return Ok(false);
-                } else {
-                    return Err(e).context("Failed to open CURRENT file");
-                }
-            }
+        let current = match read_current_version(&self.path)? {
+            Some(version) => version.max_sequence_number,
+            None if !read_only => return Ok(false),
+            None => bail!("Failed to open database: CURRENT file is missing"),
         };
-        let current = current_file.read_u32::<BE>()?;
-        drop(current_file);
 
         let mut deleted_files = HashSet::new();
         for entry in entries {

turbopack/crates/turbo-persistence/src/lib.rs

@@ -30,7 +30,10 @@ mod tests;
 
 pub use arc_bytes::ArcBytes;
 pub use compression::checksum_block;
-pub use db::{CommitStats, CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
+pub use db::{
+    CommitStats, CompactConfig, CurrentDbVersion, MetaFileEntryInfo, MetaFileInfo,
+    TurboPersistence, read_current_version,
+};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum FamilyKind {

turbopack/crates/turbo-persistence/src/tests.rs

@@ -6,7 +6,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use crate::{
     DbConfig, FamilyConfig, FamilyKind,
     constants::{MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
-    db::{CompactConfig, TurboPersistence},
+    db::{CompactConfig, TurboPersistence, read_current_version},
     parallel_scheduler::ParallelScheduler,
     write_batch::WriteBatch,
 };
@@ -2229,3 +2229,38 @@ fn stale_current_next_is_recovered() -> Result<()> {
 
     Ok(())
 }
+
+/// `CURRENT` round-trips through JSON, recording both the sequence number and the commit time.
+#[test]
+fn current_file_is_json_with_commit_time() -> Result<()> {
+    use crate::parallel_scheduler::SerialScheduler;
+
+    let tempdir = tempfile::tempdir()?;
+    let path = tempdir.path();
+
+    let before = jiff::Timestamp::now();
+    {
+        let db = TurboPersistence::<SerialScheduler, 1>::open(path.to_path_buf())?;
+        let batch = db.write_batch()?;
+        batch.put(0, vec![1u8], vec![42u8].into())?;
+        db.commit_write_batch(batch)?;
+        db.shutdown()?;
+    }
+    let after = jiff::Timestamp::now();
+
+    // next.js parses `CURRENT` without going through this crate, so these field names are a
+    // public contract.
+    let raw = fs::read_to_string(path.join("CURRENT"))?;
+    assert!(raw.contains("max_sequence_number"), "got: {raw}");
+    assert!(raw.contains("commit_time"), "got: {raw}");
+
+    let version = read_current_version(path)?.expect("CURRENT should exist");
+    assert!(version.max_sequence_number > 0);
+    assert!(
+        version.commit_time >= before && version.commit_time <= after,
+        "commit_time {} outside [{before}, {after}]",
+        version.commit_time
+    );
+
+    Ok(())
+}

turbopack/crates/turbo-tasks-backend/Cargo.toml

@@ -45,6 +45,7 @@ dashmap = { workspace = true, features = ["raw-api"]}
 fs-err = { workspace = true }
 hashbrown = { workspace = true, features = ["raw"] }
 indexmap = { workspace = true }
+jiff = "0.2.10"
 lzzzz = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 rand = { workspace = true }